Use docker containers to test the chat app. (ServiceWeaver#641)

Recall that the chat app implements a simple online messaging forum. It
persists chat threads in a database. Before this PR, the chat app used
MySQL for production deployments and used SQLite for local deployments
and tests.

This PR removes the use of SQLite and instead always uses MySQL, even
locally and in tests. This has two advantages:

1. Running the same database in production and in tests helps discover
   more bugs. In fact, in this PR, I found a bug in the code by running
   MySQL in tests.
2. It's not always possible to run an app against SQLite. An app might
   rely on features SQLite does not offer, for example.

Always using MySQL does have some downsides. Launching a docker
container for every test is slow. The chat tests now take 24 seconds.
Most of this time comes from spinning up MySQL, not from launching the
actual container. We could put in some effort to re-use a MySQL instance
across multiple tests, though it would complicate things.

Overall, I think faking is still very important. `chat/server_test.go`
uses a `sqlStore` fake, for example. I think this will be especially
necessary for simulation testing.

> Details

- To run MySQL in tests, we leverage [testcontainers][], a Go module
  that makes it easy to run containers in tests.
- I renamed the chat database from `serviceweaver_chat_example` to just
  `chat`.
- I fixed a bug in `sqlStore` where posts were not being ordered
  correctly.

[testcontainers]: https://testcontainers.com/

examples/chat/README.md

@@ -29,7 +29,7 @@ $ docker run \
  --detach \
  --name mysql \
  --env MYSQL_ROOT_PASSWORD="password" \
- --env MYSQL_DATABASE="serviceweaver_chat_example" \
+ --env MYSQL_DATABASE="chat" \
  --volume="$(realpath chat.sql):/app/chat.sql" \
  --publish 127.0.0.1:3306:3306 \
  mysql

examples/chat/chat.sql

@@ -12,7 +12,7 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 
-USE serviceweaver_chat_example;
+USE chat;
 
 CREATE TABLE IF NOT EXISTS threads (
  thread INTEGER AUTO_INCREMENT PRIMARY KEY,

examples/chat/sqlstore.go

@@ -19,16 +19,13 @@ import (
  "database/sql"
  "errors"
  "fmt"
- "strings"
  "time"
 
  "github.com/ServiceWeaver/weaver"
  _ "github.com/go-sql-driver/mysql"
  _ "modernc.org/sqlite"
 )
 
-const dbName = "serviceweaver_chat_example"
-
 // sqlStore provides storage for chat application data.
 //
 // The store holds:
@@ -100,81 +97,19 @@ func (cfg *config) Validate() error {
 }
 
 func (s *sqlStore) Init(ctx context.Context) error {
- // TODO(mwhittaker): Don't use sqlite, and don't initialize the database
- // within Init. Just connect to the database listed in the config.
  cfg := s.Config()
-
- var db *sql.DB
- var err error
  if cfg.Driver == "" {
- cfg.Driver = "sqlite"
- db, err = sql.Open(cfg.Driver, ":memory:")
- } else {
- // Ensure chat database exists.
- ensureDB := func() error {
- db_admin, err := sql.Open(cfg.Driver, cfg.URI)
- if err != nil {
- return fmt.Errorf("error opening %q URI %q: %w", cfg.Driver, cfg.URI, err)
- }
- defer db_admin.Close()
- _, err = db_admin.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbName))
- return err
- }
- if err := ensureDB(); err != nil {
- return fmt.Errorf("error creating %q database %s%s: %w", cfg.Driver, cfg.URI, dbName, err)
- }
- db, err = sql.Open(cfg.Driver, cfg.URI+dbName)
+ return fmt.Errorf("missing database driver in config")
+ }
+ if cfg.URI == "" {
+ return fmt.Errorf("missing database URI in config")
  }
+ db, err := sql.Open(cfg.Driver, cfg.URI)
  if err != nil {
- return fmt.Errorf("error opening %q database %s%s: %w", cfg.Driver, cfg.URI, dbName, err)
+ return fmt.Errorf("error opening %q database %s: %w", cfg.Driver, cfg.URI, err)
  }
-
- // Ensure chat tables exist.
- for _, q := range []struct {
- query string
- errOK bool
- }{
- {query: `CREATE TABLE IF NOT EXISTS threads (
- thread INTEGER AUTO_INCREMENT PRIMARY KEY,
- creator VARCHAR(256) NOT NULL
- )`,
- },
- {query: `CREATE INDEX thread ON threads (thread)`, errOK: true},
- {query: `CREATE TABLE IF NOT EXISTS userthreads (
- user VARCHAR(256) NOT NULL,
- thread INTEGER NOT NULL,
- CONSTRAINT uthread FOREIGN KEY(thread) REFERENCES threads(thread)
- )`,
- },
- {query: `CREATE INDEX uthread ON userthreads (thread)`, errOK: true},
- {query: `CREATE TABLE IF NOT EXISTS images (
- id INTEGER AUTO_INCREMENT PRIMARY KEY,
- image BLOB NOT NULL
- )`,
- },
- {query: `CREATE INDEX image ON images (id)`, errOK: true},
- {query: `CREATE TABLE IF NOT EXISTS posts (
- post INTEGER AUTO_INCREMENT PRIMARY KEY,
- thread INTEGER NOT NULL,
- creator VARCHAR(256) NOT NULL,
- time INTEGER NOT NULL,
- text TEXT NOT NULL,
- imageid INTEGER,
- CONSTRAINT pthread FOREIGN KEY (thread) REFERENCES threads(thread),
- CONSTRAINT pimageid FOREIGN KEY (imageid) REFERENCES images(id)
- )`,
- },
- } {
- query := q.query
- if cfg.Driver == "sqlite" {
- // sqlite does not work with AUTO_INCREMENT specified on primary keys.
- query = strings.ReplaceAll(query, "AUTO_INCREMENT PRIMARY KEY", "PRIMARY KEY")
- }
-
- _, err = db.ExecContext(ctx, query)
- if err != nil && !q.errOK {
- return fmt.Errorf("error initializing %q database %s%s with query %q: %w", cfg.Driver, cfg.URI, dbName, q.query, err)
- }
+ if err := db.Ping(); err != nil {
+ return fmt.Errorf("error pinging %q database %s: %w", cfg.Driver, cfg.URI, err)
  }
  s.db = db
  return nil
@@ -253,7 +188,7 @@ SELECT u.thread, p.post, p.creator, p.time, p.text, p.imageid
 FROM userthreads AS u
 JOIN posts as p ON p.thread=u.thread
 WHERE u.user=?
-ORDER BY u.thread DESC;
+ORDER BY p.time, u.thread DESC;
 `
  rows, err := s.db.QueryContext(ctx, query, user)
  if err != nil {

examples/chat/sqlstore_test.go

@@ -23,12 +23,56 @@ import (
 
  "github.com/ServiceWeaver/weaver/weavertest"
  "github.com/google/go-cmp/cmp"
+ "github.com/testcontainers/testcontainers-go"
+ "github.com/testcontainers/testcontainers-go/modules/mysql"
 )
 
+// newMySQL launches a new, fully initialized MySQL instance suitable for use
+// with a sqlStore. newMySQL returns the connection string to connect to the
+// MySQL instance, and it tears down the instance when the provided test ends.
+func newMySQL(t *testing.T, ctx context.Context) string {
+ t.Helper()
+
+ // Create the container.
+ container, err := mysql.RunContainer(ctx,
+ testcontainers.WithImage("mysql"),
+ mysql.WithDatabase("chat"),
+ mysql.WithScripts("chat.sql"),
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Clean up the container when the test ends.
+ t.Cleanup(func() {
+ if err := container.Terminate(ctx); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ // Return the connection string.
+ addr, err := container.ConnectionString(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return addr
+}
+
 func TestFeed(t *testing.T) {
- weavertest.Local.Test(t, func(t *testing.T, store SQLStore) {
- ctx := context.Background()
+ testcontainers.SkipIfProviderIsNotHealthy(t)
 
+ // Start a MySQL instance.
+ ctx := context.Background()
+ addr := newMySQL(t, ctx)
+
+ // Run the test against the instance.
+ runner := weavertest.Local
+ runner.Config = fmt.Sprintf(`
+ ["github.com/ServiceWeaver/weaver/examples/chat/SQLStore"]
+ db_driver = "mysql"
+ db_uri = %q
+ `, addr)
+ runner.Test(t, func(t *testing.T, store SQLStore) {
  // Run the test.
  makeThread := func(user, msg string, others ...string) ThreadID {
  tid, err := store.CreateThread(ctx, user, time.Now(), others, msg, nil)
@@ -56,7 +100,7 @@ func TestFeed(t *testing.T) {
  post(t1, "alice", "msg3")
  post(t2, "ted", "msg4")
 
- const thread1 = "alice:msg3 bob:msg1"
+ const thread1 = "bob:msg1 alice:msg3"
  const thread2 = "ted:msg2 ted:msg4"
  for user, expect := range map[string][]string{
  "bob": {thread2, thread1},
@@ -81,7 +125,20 @@ func TestFeed(t *testing.T) {
 }
 
 func TestImage(t *testing.T) {
- weavertest.Local.Test(t, func(t *testing.T, store SQLStore) {
+ testcontainers.SkipIfProviderIsNotHealthy(t)
+
+ // Start a MySQL instance.
+ ctx := context.Background()
+ addr := newMySQL(t, ctx)
+
+ // Run the test against the instance.
+ runner := weavertest.Local
+ runner.Config = fmt.Sprintf(`
+ ["github.com/ServiceWeaver/weaver/examples/chat/SQLStore"]
+ db_driver = "mysql"
+ db_uri = %q
+ `, addr)
+ runner.Test(t, func(t *testing.T, store SQLStore) {
  ctx := context.Background()
 
  // Create thread with an image in the initial post.

examples/chat/weaver.toml

@@ -14,4 +14,4 @@ listeners.chat = {public_hostname = "chat.example.com"}
 
 ["github.com/ServiceWeaver/weaver/examples/chat/SQLStore"]
 db_driver = "mysql"
-db_uri = "root:password@tcp(localhost:3306)/"
+db_uri = "root:password@tcp(localhost:3306)/chat"

go.mod

@@ -7,14 +7,16 @@ require (
  github.com/DataDog/hyperloglog v0.0.0-20220804205443-1806d9b66146
  github.com/alecthomas/chroma/v2 v2.2.0
  github.com/fsnotify/fsnotify v1.6.0
- github.com/go-sql-driver/mysql v1.7.0
+ github.com/go-sql-driver/mysql v1.7.1
  github.com/google/cel-go v0.17.1
  github.com/google/go-cmp v0.5.9
  github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8
- github.com/google/uuid v1.3.0
+ github.com/google/uuid v1.3.1
  github.com/hashicorp/golang-lru/v2 v2.0.1
  github.com/lightstep/varopt v1.3.0
  github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
+ github.com/testcontainers/testcontainers-go v0.24.1
+ github.com/testcontainers/testcontainers-go/modules/mysql v0.24.1
  github.com/yuin/goldmark v1.4.15
  github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
@@ -26,30 +28,64 @@ require (
  golang.org/x/image v0.5.0
  golang.org/x/sync v0.3.0
  golang.org/x/term v0.10.0
- golang.org/x/text v0.9.0
+ golang.org/x/text v0.11.0
  golang.org/x/tools v0.11.0
  google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753
  google.golang.org/protobuf v1.31.0
  modernc.org/sqlite v1.24.0
 )
 
 require (
+ dario.cat/mergo v1.0.0 // indirect
+ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
  github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect
+ github.com/Microsoft/go-winio v0.6.1 // indirect
+ github.com/Microsoft/hcsshim v0.11.0 // indirect
  github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+ github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+ github.com/containerd/containerd v1.7.6 // indirect
+ github.com/cpuguy83/dockercfg v0.3.1 // indirect
  github.com/dlclark/regexp2 v1.7.0 // indirect
+ github.com/docker/distribution v2.8.2+incompatible // indirect
+ github.com/docker/docker v24.0.6+incompatible // indirect
+ github.com/docker/go-connections v0.4.0 // indirect
+ github.com/docker/go-units v0.5.0 // indirect
  github.com/dustin/go-humanize v1.0.1 // indirect
  github.com/dustin/randbo v0.0.0-20140428231429-7f1b564ca724 // indirect
  github.com/felixge/httpsnoop v1.0.3 // indirect
  github.com/go-logr/logr v1.2.4 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-ole/go-ole v1.2.6 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/protobuf v1.5.3 // indirect
  github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+ github.com/klauspost/compress v1.16.0 // indirect
+ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+ github.com/magiconair/properties v1.8.7 // indirect
  github.com/mattn/go-isatty v0.0.16 // indirect
+ github.com/moby/patternmatcher v0.5.0 // indirect
+ github.com/moby/sys/sequential v0.5.0 // indirect
+ github.com/moby/term v0.5.0 // indirect
+ github.com/morikuni/aec v1.0.0 // indirect
+ github.com/opencontainers/go-digest v1.0.0 // indirect
+ github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
+ github.com/opencontainers/runc v1.1.5 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
  github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+ github.com/shirou/gopsutil/v3 v3.23.7 // indirect
+ github.com/shoenig/go-m1cpu v0.1.6 // indirect
+ github.com/sirupsen/logrus v1.9.0 // indirect
  github.com/stoewer/go-strcase v1.2.0 // indirect
+ github.com/tklauser/go-sysconf v0.3.11 // indirect
+ github.com/tklauser/numcpus v0.6.0 // indirect
+ github.com/yusufpapurcu/wmi v1.2.3 // indirect
  go.opentelemetry.io/otel/metric v1.16.0 // indirect
  golang.org/x/mod v0.12.0 // indirect
- golang.org/x/sys v0.10.0 // indirect
+ golang.org/x/net v0.12.0 // indirect
+ golang.org/x/sys v0.11.0 // indirect
  google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
+ google.golang.org/grpc v1.57.0 // indirect
  lukechampine.com/uint128 v1.2.0 // indirect
  modernc.org/cc/v3 v3.40.0 // indirect
  modernc.org/ccgo/v3 v3.16.13 // indirect