From 02247b215b2fa1c1c18f651561092290ed2a5058 Mon Sep 17 00:00:00 2001 From: Paco Esteban Date: Thu, 2 May 2024 16:38:16 +0200 Subject: [PATCH] refactor: allow specific sqlite engine for OpenBSD (#780) * remove dependencies that use syscall.Syscall* OpenBSD will be removing direct access to `syscall(2)` soon. Shiori will stop working because of this, as some of its dependencies rely heavily on the use of `syscall.Syscall*`, which ends up using `syscall(2)`. This commit removes those dependencies by reverting back to use github.com/mattn/go-sqlite3 instead of modernc.org/sqlite to deal with the sqlite database backend. * add ~emersion/go-sqlite3-fts5 * enable cgo to check ci * fts5 build flag * split sqlite logic using build flags * disable cgo again * added ci test for bsd systems * remove openbsd ci * Revert "remove openbsd ci" This reverts commit f394148385fe10aa72b946e678766fb16c782636. * fix makefile go_test_flags --------- Co-authored-by: Felipe M Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> --- .github/workflows/_test.yml | 35 ++++++++++++++++++++++++++++- Makefile | 10 ++++----- go.mod | 2 ++ go.sum | 2 ++ internal/database/sqlite.go | 12 ---------- internal/database/sqlite_noncgo.go | 25 +++++++++++++++++++++ internal/database/sqlite_openbsd.go | 26 +++++++++++++++++++++ 7 files changed, 94 insertions(+), 18 deletions(-) create mode 100644 internal/database/sqlite_noncgo.go create mode 100644 internal/database/sqlite_openbsd.go diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 715b1d6ea..1eaf40cea 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -89,8 +89,41 @@ jobs: golangci-lint.cache-{interval_number}- golangci-lint.cache- - - run: make unittest GO_TEST_FLAGS="-tags test_sqlite_only" + - run: make unittest GO_TEST_FLAGS="-tags test_sqlite_only -race -v -count=1" env: CGO_ENABLED: 1 # go test -race requires cgo - run: go build -tags osusergo,netgo -ldflags="-s -w -X main.version=$(git describe --tags) -X main.date=$(date --iso-8601=seconds)" + + # Please note BSD support is offered on a best-effort basis, this check is not blocking but for us to be aware of issues. + # This test also does not take into consideration the go version specified in the go.mod file and just uses the + # latest version available in the openbsd package repository. + test-bsd: + runs-on: ubuntu-latest + strategy: + matrix: + os: + - name: openbsd + architecture: x86-64 + version: '7.5' + + - name: openbsd + architecture: arm64 + version: '7.5' + steps: + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + + - name: Test on ${{ matrix.os.name }} + uses: cross-platform-actions/action@b2e15da1e667187766fff4945d20b98ac7055576 # v0.24.0 + with: + environment_variables: GO_VERSION + operating_system: ${{ matrix.os.name }} + architecture: ${{ matrix.os.architecture }} + version: ${{ matrix.os.version }} + shell: bash + memory: 1G + cpu_count: 1 + run: | + sudo pkg_add -u + sudo pkg_add gmake git go + gmake unittest GO_TEST_FLAGS="-tags test_sqlite_only -v -count=1" diff --git a/Makefile b/Makefile index 59a2044de..efd8e36a9 100644 --- a/Makefile +++ b/Makefile @@ -4,17 +4,17 @@ BASH ?= $(shell command -v bash 2> /dev/null) # Development SHIORI_DIR ?= dev-data -# Testing -override GO_TEST_FLAGS += -v -race -count=1 -covermode=atomic -coverprofile=coverage.out -GOTESTFMT_FLAGS ?= - # Build CGO_ENABLED ?= 0 BUILD_TIME := $(shell date -u +%Y%m%d.%H%M%S) BUILD_HASH := $(shell git describe --tags) -BUILD_TAGS ?= osusergo,netgo +BUILD_TAGS ?= osusergo,netgo,fts5 LDFLAGS += -s -w -X main.version=$(BUILD_HASH) -X main.date=$(BUILD_TIME) +# Testing +GO_TEST_FLAGS ?= -v -race -count=1 -tags $(BUILD_TAGS) -covermode=atomic -coverprofile=coverage.out +GOTESTFMT_FLAGS ?= + # Development GIN_MODE ?= debug SHIORI_DEVELOPMENT ?= true diff --git a/go.mod b/go.mod index f245c1059..6905680f4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ module github.com/go-shiori/shiori go 1.22.2 require ( + git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc github.com/PuerkitoBio/goquery v1.9.1 github.com/blang/semver v3.5.1+incompatible github.com/disintegration/imaging v1.6.2 @@ -20,6 +21,7 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/julienschmidt/httprouter v1.3.0 github.com/lib/pq v1.10.9 + github.com/mattn/go-sqlite3 v1.14.22 github.com/muesli/go-app-paths v0.2.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 9e1de685c..99687a864 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc h1:+y3OijpLl4rgbFsqMBmYUTCsGCkxQUWpWaqfS8j9Ygc= +git.sr.ht/~emersion/go-sqlite3-fts5 v0.0.0-20230217131031-f2c8767594fc/go.mod h1:PCl1xjl7iC6x35TKKubKRyo/3TT0dGI66jyNI6vmYnU= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= diff --git a/internal/database/sqlite.go b/internal/database/sqlite.go index 43f931794..4cc117b12 100644 --- a/internal/database/sqlite.go +++ b/internal/database/sqlite.go @@ -78,18 +78,6 @@ type tagContent struct { model.Tag } -// OpenSQLiteDatabase creates and open connection to new SQLite3 database. -func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) { - // Open database - db, err := sqlx.ConnectContext(ctx, "sqlite", databasePath) - if err != nil { - return nil, errors.WithStack(err) - } - - sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}} - return sqliteDB, nil -} - // DBX returns the underlying sqlx.DB object func (db *SQLiteDatabase) DBx() sqlx.DB { return db.DB diff --git a/internal/database/sqlite_noncgo.go b/internal/database/sqlite_noncgo.go new file mode 100644 index 000000000..4c870e6a1 --- /dev/null +++ b/internal/database/sqlite_noncgo.go @@ -0,0 +1,25 @@ +//go:build linux || windows || darwin +// +build linux windows darwin + +package database + +import ( + "context" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + + _ "modernc.org/sqlite" +) + +// OpenSQLiteDatabase creates and open connection to new SQLite3 database. +func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) { + // Open database + db, err := sqlx.ConnectContext(ctx, "sqlite", databasePath) + if err != nil { + return nil, errors.WithStack(err) + } + + sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}} + return sqliteDB, nil +} diff --git a/internal/database/sqlite_openbsd.go b/internal/database/sqlite_openbsd.go new file mode 100644 index 000000000..e376568ec --- /dev/null +++ b/internal/database/sqlite_openbsd.go @@ -0,0 +1,26 @@ +//go:build openbsd +// +build openbsd + +package database + +import ( + "context" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + + _ "git.sr.ht/~emersion/go-sqlite3-fts5" + _ "github.com/mattn/go-sqlite3" +) + +// OpenSQLiteDatabase creates and open connection to new SQLite3 database. +func OpenSQLiteDatabase(ctx context.Context, databasePath string) (sqliteDB *SQLiteDatabase, err error) { + // Open database + db, err := sqlx.ConnectContext(ctx, "sqlite3", databasePath) + if err != nil { + return nil, errors.WithStack(err) + } + + sqliteDB = &SQLiteDatabase{dbbase: dbbase{*db}} + return sqliteDB, nil +}