From 9b566382a600d4e9bd83fa71f32f3e021fac2ecc Mon Sep 17 00:00:00 2001 From: Stany MARCEL Date: Tue, 19 May 2020 09:31:29 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9BFix=20bug=20on=20bookmark=20ins?= =?UTF-8?q?ert=20for=20pgsql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bookmark id auto increment did not permit to remove a bookmak and then insert a new one with postgresql. Signed-off-by: Stany MARCEL --- internal/database/pg.go | 49 ++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/internal/database/pg.go b/internal/database/pg.go index 37512006d..d09ffda79 100644 --- a/internal/database/pg.go +++ b/internal/database/pg.go @@ -68,11 +68,9 @@ func OpenPGDatabase(connString string) (pgDB *PGDatabase, err error) { CONSTRAINT tag_name_UNIQUE UNIQUE (name))`) tx.MustExec(`CREATE TABLE IF NOT EXISTS bookmark_tag( - bookmark_id INT NOT NULL, - tag_id INT NOT NULL, - PRIMARY KEY(bookmark_id, tag_id), - CONSTRAINT bookmark_tag_bookmark_id_FK FOREIGN KEY (bookmark_id) REFERENCES bookmark (id), - CONSTRAINT bookmark_tag_tag_id_FK FOREIGN KEY (tag_id) REFERENCES tag (id))`) + bookmark_id SERIAL REFERENCES bookmark(id) ON DELETE CASCADE, + tag_id SERIAL REFERENCES tag(id) ON DELETE CASCADE, + PRIMARY KEY(bookmark_id, tag_id))`) // Create indices tx.MustExec(`CREATE INDEX IF NOT EXISTS bookmark_tag_bookmark_id_FK ON bookmark_tag (bookmark_id)`) @@ -110,14 +108,15 @@ func (db *PGDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []model (url, title, excerpt, author, public, content, html, modified) VALUES($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT(url) DO UPDATE SET - url = $1, - title = $2, - excerpt = $3, - author = $4, - public = $5, - content = $6, - html = $7, - modified = $8`) + url = $1, + title = $2, + excerpt = $3, + author = $4, + public = $5, + content = $6, + html = $7, + modified = $8 + RETURNING id`) checkError(err) stmtGetTag, err := tx.Preparex(`SELECT id FROM tag WHERE name = $1`) @@ -157,7 +156,7 @@ func (db *PGDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []model book.Modified = modifiedTime // Save bookmark - stmtInsertBook.MustExec( + stmtInsertBook.Get(&(book.ID), book.URL, book.Title, book.Excerpt, book.Author, book.Public, book.Content, book.HTML, book.Modified) @@ -316,6 +315,9 @@ func (db *PGDatabase) GetBookmarks(opts GetBookmarksOptions) ([]model.Bookmark, // Expand query, because some of the args might be an array query, args, err := sqlx.Named(query, arg) + if err != nil { + return nil, fmt.Errorf("failed to create named query: %v", err) + } query, args, err = sqlx.In(query, args...) if err != nil { return nil, fmt.Errorf("failed to expand query: %v", err) @@ -432,6 +434,9 @@ func (db *PGDatabase) GetBookmarksCount(opts GetBookmarksOptions) (int, error) { // Expand query, because some of the args might be an array query, args, err := sqlx.Named(query, arg) + if err != nil { + return 0, fmt.Errorf("failed to create named query: %v", err) + } query, args, err = sqlx.In(query, args...) if err != nil { return 0, fmt.Errorf("failed to expand query: %v", err) @@ -468,21 +473,16 @@ func (db *PGDatabase) DeleteBookmarks(ids ...int) (err error) { // Prepare queries delBookmark := `DELETE FROM bookmark` - delBookmarkTag := `DELETE FROM bookmark_tag` // Delete bookmark(s) if len(ids) == 0 { - tx.MustExec(delBookmarkTag) tx.MustExec(delBookmark) } else { delBookmark += ` WHERE id = $1` - delBookmarkTag += ` WHERE bookmark_id = $1` stmtDelBookmark, _ := tx.Preparex(delBookmark) - stmtDelBookmarkTag, _ := tx.Preparex(delBookmarkTag) for _, id := range ids { - stmtDelBookmarkTag.MustExec(id) stmtDelBookmark.MustExec(id) } } @@ -626,13 +626,6 @@ func (db *PGDatabase) RenameTag(id int, newName string) error { // CreateNewID creates new ID for specified table func (db *PGDatabase) CreateNewID(table string) (int, error) { - var tableID int - query := fmt.Sprintf(`SELECT last_value from %s_id_seq;`, table) - - err := db.Get(&tableID, query) - if err != nil && err != sql.ErrNoRows { - return -1, err - } - - return tableID, nil + // dummy id as not used on postgresql with autoincrement serial + return 1, nil } From 0f376042b55e5bbc55ed2966e3372646f42325ca Mon Sep 17 00:00:00 2001 From: Stany MARCEL Date: Tue, 19 May 2020 09:38:16 +0200 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9BRemove=20invalid=20UTF8=20runes?= =?UTF-8?q?=20in=20strings=20before=20db=20insert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stany MARCEL --- internal/database/database.go | 12 ++++++++++++ internal/database/mysql.go | 13 ++++++++++--- internal/database/pg.go | 13 ++++++++++--- internal/database/sqlite.go | 31 +++++++++++++++++++++++++------ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/internal/database/database.go b/internal/database/database.go index 44819d1fa..b3eed8851 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -2,6 +2,8 @@ package database import ( "database/sql" + "strings" + "unicode/utf8" "github.com/go-shiori/shiori/internal/model" ) @@ -80,3 +82,13 @@ func checkError(err error) { panic(err) } } + +func sanitizeString(str string) string { + fixUtf := func(r rune) rune { + if r == utf8.RuneError { + return -1 + } + return r + } + return strings.Map(fixUtf, str) +} diff --git a/internal/database/mysql.go b/internal/database/mysql.go index 6fb622561..a650d4a88 100644 --- a/internal/database/mysql.go +++ b/internal/database/mysql.go @@ -162,8 +162,14 @@ func (db *MySQLDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []mo // Save bookmark stmtInsertBook.MustExec(book.ID, - book.URL, book.Title, book.Excerpt, book.Author, - book.Public, book.Content, book.HTML, book.Modified) + sanitizeString(book.URL), + sanitizeString(book.Title), + sanitizeString(book.Excerpt), + sanitizeString(book.Author), + book.Public, + sanitizeString(book.Content), + sanitizeString(book.HTML), + book.Modified) // Save book tags newTags := []model.Tag{} @@ -175,7 +181,8 @@ func (db *MySQLDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []mo } // Normalize tag name - tagName := strings.ToLower(tag.Name) + tagName := sanitizeString(tag.Name) + tagName = strings.ToLower(tagName) tagName = strings.Join(strings.Fields(tagName), " ") // If tag doesn't have any ID, fetch it from database diff --git a/internal/database/pg.go b/internal/database/pg.go index d09ffda79..b9d7898d6 100644 --- a/internal/database/pg.go +++ b/internal/database/pg.go @@ -157,8 +157,14 @@ func (db *PGDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []model // Save bookmark stmtInsertBook.Get(&(book.ID), - book.URL, book.Title, book.Excerpt, book.Author, - book.Public, book.Content, book.HTML, book.Modified) + sanitizeString(book.URL), + sanitizeString(book.Title), + sanitizeString(book.Excerpt), + sanitizeString(book.Author), + book.Public, + sanitizeString(book.Content), + sanitizeString(book.HTML), + book.Modified) // Save book tags newTags := []model.Tag{} @@ -170,7 +176,8 @@ func (db *PGDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []model } // Normalize tag name - tagName := strings.ToLower(tag.Name) + tagName := sanitizeString(tag.Name) + tagName = strings.ToLower(tagName) tagName = strings.Join(strings.Fields(tagName), " ") // If tag doesn't have any ID, fetch it from database diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index ef561d5e1..6ed26e1d5 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -154,11 +154,29 @@ func (db *SQLiteDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []m // Save bookmark stmtInsertBook.MustExec(book.ID, - book.URL, book.Title, book.Excerpt, book.Author, book.Public, book.Modified, - book.URL, book.Title, book.Excerpt, book.Author, book.Public, book.Modified) - - stmtUpdateBookContent.MustExec(book.Title, book.Content, book.HTML, book.ID) - stmtInsertBookContent.MustExec(book.ID, book.Title, book.Content, book.HTML) + sanitizeString(book.URL), + sanitizeString(book.Title), + sanitizeString(book.Excerpt), + sanitizeString(book.Author), + book.Public, + book.Modified, + sanitizeString(book.URL), + sanitizeString(book.Title), + sanitizeString(book.Excerpt), + sanitizeString(book.Author), + book.Public, + book.Modified) + + stmtUpdateBookContent.MustExec( + sanitizeString(book.Title), + sanitizeString(book.Content), + sanitizeString(book.HTML), + book.ID) + stmtInsertBookContent.MustExec( + book.ID, + sanitizeString(book.Title), + sanitizeString(book.Content), + sanitizeString(book.HTML)) // Save book tags newTags := []model.Tag{} @@ -170,7 +188,8 @@ func (db *SQLiteDatabase) SaveBookmarks(bookmarks ...model.Bookmark) (result []m } // Normalize tag name - tagName := strings.ToLower(tag.Name) + tagName := sanitizeString(tag.Name) + tagName = strings.ToLower(tagName) tagName = strings.Join(strings.Fields(tagName), " ") // If tag doesn't have any ID, fetch it from database