From 8110f8cda53b143050b2b6e08c9c042e7a16c59c Mon Sep 17 00:00:00 2001 From: Bhupesh Kumar Date: Tue, 9 Dec 2025 12:04:21 +0530 Subject: [PATCH 1/5] fixed --- cmd/api/app_test.go | 3 +- go.mod | 12 +-- go.sum | 44 ++++----- gtfsdb/bulk_insert_test.go | 2 +- gtfsdb/client.go | 2 +- gtfsdb/conditional_import_test.go | 2 +- gtfsdb/connection_pool_test.go | 4 +- gtfsdb/db.go | 10 ++ gtfsdb/error_handling_test.go | 2 +- gtfsdb/helpers.go | 4 +- gtfsdb/nil_shape_test.go | 2 +- gtfsdb/query.sql | 25 +++++ gtfsdb/query.sql.go | 66 +++++++++++++ gtfsdb/schema.sql | 72 ++++++++++++++ gtfsdb/sqlite_performance_test.go | 4 +- internal/gtfs/gtfs_manager.go | 2 +- internal/gtfs/route_search.go | 46 +++++++++ internal/restapi/route_search_handler.go | 94 +++++++++++++++++++ internal/restapi/route_search_handler_test.go | 69 ++++++++++++++ internal/restapi/routes.go | 1 + internal/utils/filters_test.go | 2 +- 21 files changed, 417 insertions(+), 51 deletions(-) create mode 100644 internal/gtfs/route_search.go create mode 100644 internal/restapi/route_search_handler.go create mode 100644 internal/restapi/route_search_handler_test.go diff --git a/cmd/api/app_test.go b/cmd/api/app_test.go index 2db15ae9..8ea2893d 100644 --- a/cmd/api/app_test.go +++ b/cmd/api/app_test.go @@ -9,11 +9,11 @@ import ( "testing" "time" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/gtfs" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestParseAPIKeys(t *testing.T) { @@ -445,6 +445,7 @@ func TestBuildApplicationWithConfigFile(t *testing.T) { // Convert to absolute path to avoid path traversal validation issues absTestDataPath, err := filepath.Abs(testDataPath) require.NoError(t, err) + absTestDataPath = filepath.ToSlash(absTestDataPath) // Create a test config file that uses the test data testConfigPath := filepath.Join("..", "..", "testdata", "config_test_build.json") diff --git a/go.mod b/go.mod index a09d4aa0..42897ed1 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/OneBusAway/go-gtfs v1.1.0 github.com/davecgh/go-spew v1.1.1 github.com/klauspost/compress v1.18.0 - github.com/mattn/go-sqlite3 v1.14.24 github.com/stretchr/testify v1.10.0 github.com/tidwall/rtree v1.10.0 github.com/twpayne/go-polyline v1.1.1 golang.org/x/time v0.12.0 + modernc.org/sqlite v1.40.1 ) require ( @@ -31,10 +31,6 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 // indirect - github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 // indirect - github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 // indirect - github.com/paulmach/go.geojson v1.5.0 // indirect github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect @@ -49,7 +45,6 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/tidwall/geoindex v1.7.0 // indirect - github.com/valyala/fastjson v1.6.4 // indirect github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect go.uber.org/atomic v1.11.0 // indirect @@ -59,7 +54,7 @@ require ( golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.27.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect @@ -67,10 +62,9 @@ require ( google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.66.3 // indirect + modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.37.0 // indirect ) tool github.com/sqlc-dev/sqlc/cmd/sqlc diff --git a/go.sum b/go.sum index 78d46807..676d8d17 100644 --- a/go.sum +++ b/go.sum @@ -60,18 +60,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 h1:ei2LAhpj7frAPBzbjqTA9ICXi7H2KhBXlYzO0WwP8hI= -github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1/go.mod h1:WsjXLsxSQc+KfBJ7APQhk3yz+4DztzMYARQ+EfxKYSQ= -github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 h1:5RbZdJNYJbywbCwaQOzHUckzJ+F0w54imKZhkL0eUB0= -github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868/go.mod h1:qNytqg0F2xv/NCYHECvfjr/QPioK94q2rhPTC7NI4KA= -github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 h1:xEQhft28izewbL3fnF10Yj8zSNu2r+lW157GQq+eaaY= -github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8/go.mod h1:zYAfZRXtDhc4Tq+LYrvyksefggNiViL2pKMRUrGvUbE= -github.com/paulmach/go.geojson v1.5.0 h1:7mhpMK89SQdHFcEGomT7/LuJhwhEgfmpWYVlVmLEdQw= -github.com/paulmach/go.geojson v1.5.0/go.mod h1:DgdUy2rRVDDVgKqrjMe2vZAHMfhDTrjVKt3LmHIXGbU= github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls= github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -121,8 +111,6 @@ github.com/tidwall/rtree v1.10.0 h1:+EcI8fboEaW1L3/9oW/6AMoQ8HiEIHyR7bQOGnmz4Mg= github.com/tidwall/rtree v1.10.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/twpayne/go-polyline v1.1.1 h1:/tSF1BR7rN4HWj4XKqvRUNrCiYVMCvywxTFVofvDV0w= github.com/twpayne/go-polyline v1.1.1/go.mod h1:ybd9IWWivW/rlXPXuuckeKUyF3yrIim+iqA7kSl4NFY= -github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= -github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfPe7z7go8Dvv1AJQDI3eQ/5xith3q2mFlo= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07/go.mod h1:Ak17IJ037caFp4jpCw/iQQ7/W74Sqpb1YuKJU6HTKfM= github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= @@ -160,8 +148,8 @@ golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZv golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc= golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= @@ -171,8 +159,8 @@ golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= @@ -181,8 +169,8 @@ golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= @@ -208,18 +196,18 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= -modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= -modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= -modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -228,8 +216,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= -modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= +modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY= +modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/gtfsdb/bulk_insert_test.go b/gtfsdb/bulk_insert_test.go index 3b6b3ee6..f8688de1 100644 --- a/gtfsdb/bulk_insert_test.go +++ b/gtfsdb/bulk_insert_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestBulkInsertStopTimes(t *testing.T) { diff --git a/gtfsdb/client.go b/gtfsdb/client.go index bc269bd7..f22156b4 100644 --- a/gtfsdb/client.go +++ b/gtfsdb/client.go @@ -10,7 +10,7 @@ import ( "os" "time" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) // Client is the main entry point for the library diff --git a/gtfsdb/conditional_import_test.go b/gtfsdb/conditional_import_test.go index 2462b58b..321dfaf4 100644 --- a/gtfsdb/conditional_import_test.go +++ b/gtfsdb/conditional_import_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) // getTestFixturePath returns the absolute path to a fixture file in the testdata directory diff --git a/gtfsdb/connection_pool_test.go b/gtfsdb/connection_pool_test.go index 9784848e..e2491294 100644 --- a/gtfsdb/connection_pool_test.go +++ b/gtfsdb/connection_pool_test.go @@ -5,10 +5,10 @@ import ( "database/sql" "testing" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestDatabaseConnectionPoolSettings(t *testing.T) { @@ -103,7 +103,7 @@ func TestConnectionLifetime(t *testing.T) { func TestConnectionPoolConfiguration(t *testing.T) { // Test the specific configuration values for in-memory databases - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") require.NoError(t, err, "Should open database") defer func() { _ = db.Close() }() diff --git a/gtfsdb/db.go b/gtfsdb/db.go index bb938e08..f178d067 100644 --- a/gtfsdb/db.go +++ b/gtfsdb/db.go @@ -258,6 +258,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.listRoutesStmt, err = db.PrepareContext(ctx, listRoutes); err != nil { return nil, fmt.Errorf("error preparing query ListRoutes: %w", err) } + if q.searchRoutesByFullTextStmt, err = db.PrepareContext(ctx, searchRoutesByFullText); err != nil { + return nil, fmt.Errorf("error preparing query SearchRoutesByFullText: %w", err) + } if q.listStopsStmt, err = db.PrepareContext(ctx, listStops); err != nil { return nil, fmt.Errorf("error preparing query ListStops: %w", err) } @@ -665,6 +668,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing listRoutesStmt: %w", cerr) } } + if q.searchRoutesByFullTextStmt != nil { + if cerr := q.searchRoutesByFullTextStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchRoutesByFullTextStmt: %w", cerr) + } + } if q.listStopsStmt != nil { if cerr := q.listStopsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing listStopsStmt: %w", cerr) @@ -802,6 +810,7 @@ type Queries struct { getTripsInBlockStmt *sql.Stmt listAgenciesStmt *sql.Stmt listRoutesStmt *sql.Stmt + searchRoutesByFullTextStmt *sql.Stmt listStopsStmt *sql.Stmt listTripsStmt *sql.Stmt updateStopDirectionStmt *sql.Stmt @@ -890,6 +899,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTripsInBlockStmt: q.getTripsInBlockStmt, listAgenciesStmt: q.listAgenciesStmt, listRoutesStmt: q.listRoutesStmt, + searchRoutesByFullTextStmt: q.searchRoutesByFullTextStmt, listStopsStmt: q.listStopsStmt, listTripsStmt: q.listTripsStmt, updateStopDirectionStmt: q.updateStopDirectionStmt, diff --git a/gtfsdb/error_handling_test.go b/gtfsdb/error_handling_test.go index ff732966..66d790ed 100644 --- a/gtfsdb/error_handling_test.go +++ b/gtfsdb/error_handling_test.go @@ -6,10 +6,10 @@ import ( "net/http/httptest" "testing" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestNewClient_InvalidConfigHandling(t *testing.T) { diff --git a/gtfsdb/helpers.go b/gtfsdb/helpers.go index bb6b329a..7259ee15 100644 --- a/gtfsdb/helpers.go +++ b/gtfsdb/helpers.go @@ -15,9 +15,9 @@ import ( "time" "github.com/OneBusAway/go-gtfs" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/logging" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) //go:embed schema.sql @@ -29,7 +29,7 @@ func createDB(config Config) (*sql.DB, error) { return nil, fmt.Errorf("test database must use in-memory storage, got path: %s", config.DBPath) } - db, err := sql.Open("sqlite3", config.DBPath) + db, err := sql.Open("sqlite", config.DBPath) if err != nil { return nil, err } diff --git a/gtfsdb/nil_shape_test.go b/gtfsdb/nil_shape_test.go index d322a2a2..6163b711 100644 --- a/gtfsdb/nil_shape_test.go +++ b/gtfsdb/nil_shape_test.go @@ -6,9 +6,9 @@ import ( "context" "testing" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) // createMinimalGTFSWithoutShapes creates a minimal GTFS zip file that has trips WITHOUT shape_id diff --git a/gtfsdb/query.sql b/gtfsdb/query.sql index c10cf71d..7fbfe57f 100644 --- a/gtfsdb/query.sql +++ b/gtfsdb/query.sql @@ -152,6 +152,31 @@ ORDER BY agency_id, id; +-- name: SearchRoutesByFullText :many +SELECT + r.id, + r.agency_id, + r.short_name, + r.long_name, + r."desc", + r.type, + r.url, + r.color, + r.text_color, + r.continuous_pickup, + r.continuous_drop_off +FROM + routes_fts + JOIN routes r ON r.rowid = routes_fts.rowid +WHERE + routes_fts MATCH @query +ORDER BY + bm25(routes_fts), + r.agency_id, + r.id +LIMIT + @limit; + -- name: GetRouteIDsForAgency :many SELECT r.id diff --git a/gtfsdb/query.sql.go b/gtfsdb/query.sql.go index 3b3d490c..053e7d3c 100644 --- a/gtfsdb/query.sql.go +++ b/gtfsdb/query.sql.go @@ -3487,6 +3487,72 @@ func (q *Queries) ListRoutes(ctx context.Context) ([]Route, error) { return items, nil } +const searchRoutesByFullText = `-- name: SearchRoutesByFullText :many +SELECT + r.id, + r.agency_id, + r.short_name, + r.long_name, + r."desc", + r.type, + r.url, + r.color, + r.text_color, + r.continuous_pickup, + r.continuous_drop_off +FROM + routes_fts + JOIN routes r ON r.rowid = routes_fts.rowid +WHERE + routes_fts MATCH ? +ORDER BY + bm25(routes_fts), + r.agency_id, + r.id +LIMIT + ? +` + +type SearchRoutesByFullTextParams struct { + Query string + Limit int64 +} + +func (q *Queries) SearchRoutesByFullText(ctx context.Context, arg SearchRoutesByFullTextParams) ([]Route, error) { + rows, err := q.query(ctx, q.searchRoutesByFullTextStmt, searchRoutesByFullText, arg.Query, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Route + for rows.Next() { + var i Route + if err := rows.Scan( + &i.ID, + &i.AgencyID, + &i.ShortName, + &i.LongName, + &i.Desc, + &i.Type, + &i.Url, + &i.Color, + &i.TextColor, + &i.ContinuousPickup, + &i.ContinuousDropOff, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listStops = `-- name: ListStops :many SELECT id, code, name, "desc", lat, lon, zone_id, url, location_type, timezone, wheelchair_boarding, platform_code, direction diff --git a/gtfsdb/schema.sql b/gtfsdb/schema.sql index e0713c60..91bc75b9 100644 --- a/gtfsdb/schema.sql +++ b/gtfsdb/schema.sql @@ -30,6 +30,78 @@ CREATE TABLE FOREIGN KEY (agency_id) REFERENCES agencies (id) ); +-- migrate +CREATE VIRTUAL TABLE IF NOT EXISTS routes_fts USING fts5 ( + id UNINDEXED, + agency_id UNINDEXED, + short_name, + long_name, + desc, + content = 'routes', + content_rowid = 'rowid' +); + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_ai AFTER INSERT ON routes BEGIN +INSERT INTO + routes_fts(rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + new.rowid, + new.id, + new.agency_id, + coalesce(new.short_name, ''), + coalesce(new.long_name, ''), + coalesce(new.desc, '') + ); +END; + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_ad AFTER DELETE ON routes BEGIN +INSERT INTO + routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + 'delete', + old.rowid, + old.id, + old.agency_id, + coalesce(old.short_name, ''), + coalesce(old.long_name, ''), + coalesce(old.desc, '') + ); +END; + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_au AFTER UPDATE ON routes BEGIN +INSERT INTO + routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + 'delete', + old.rowid, + old.id, + old.agency_id, + coalesce(old.short_name, ''), + coalesce(old.long_name, ''), + coalesce(old.desc, '') + ); +INSERT INTO + routes_fts(rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + new.rowid, + new.id, + new.agency_id, + coalesce(new.short_name, ''), + coalesce(new.long_name, ''), + coalesce(new.desc, '') + ); +END; + +-- migrate +INSERT INTO routes_fts(routes_fts) VALUES ('rebuild'); + -- migrate CREATE TABLE IF NOT EXISTS stops ( diff --git a/gtfsdb/sqlite_performance_test.go b/gtfsdb/sqlite_performance_test.go index 41ff4529..005c0fd1 100644 --- a/gtfsdb/sqlite_performance_test.go +++ b/gtfsdb/sqlite_performance_test.go @@ -7,10 +7,10 @@ import ( "path/filepath" "testing" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestSQLitePerformancePragmasApplied(t *testing.T) { @@ -218,7 +218,7 @@ func TestConfigureConnectionPoolWithDifferentConfigs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") require.NoError(t, err) defer func() { _ = db.Close() }() diff --git a/internal/gtfs/gtfs_manager.go b/internal/gtfs/gtfs_manager.go index a05f417c..4ad08e80 100644 --- a/internal/gtfs/gtfs_manager.go +++ b/internal/gtfs/gtfs_manager.go @@ -13,8 +13,8 @@ import ( "maglev.onebusaway.org/internal/utils" "github.com/OneBusAway/go-gtfs" - _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/tidwall/rtree" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) const NoRadiusLimit = -1 diff --git a/internal/gtfs/route_search.go b/internal/gtfs/route_search.go new file mode 100644 index 00000000..fbbae9f3 --- /dev/null +++ b/internal/gtfs/route_search.go @@ -0,0 +1,46 @@ +package gtfs + +import ( + "context" + "strings" + + "maglev.onebusaway.org/gtfsdb" +) + +// buildRouteSearchQuery normalizes user input into an FTS5-safe prefix search query. +func buildRouteSearchQuery(input string) string { + terms := strings.Fields(strings.ToLower(input)) + safeTerms := make([]string, 0, len(terms)) + + for _, term := range terms { + trimmed := strings.TrimSpace(term) + if trimmed == "" { + continue + } + escaped := strings.ReplaceAll(trimmed, `"`, `""`) + safeTerms = append(safeTerms, `"`+escaped+`"*`) + } + + if len(safeTerms) == 0 { + return "" + } + + return strings.Join(safeTerms, " AND ") +} + +// SearchRoutes performs a full text search against routes using SQLite FTS5. +func (manager *Manager) SearchRoutes(ctx context.Context, input string, maxCount int) ([]gtfsdb.Route, error) { + limit := maxCount + if limit <= 0 { + limit = 20 + } + if limit > 100 { + limit = 100 + } + + query := buildRouteSearchQuery(input) + return manager.GtfsDB.Queries.SearchRoutesByFullText(ctx, gtfsdb.SearchRoutesByFullTextParams{ + Query: query, + Limit: int64(limit), + }) +} diff --git a/internal/restapi/route_search_handler.go b/internal/restapi/route_search_handler.go new file mode 100644 index 00000000..35177d68 --- /dev/null +++ b/internal/restapi/route_search_handler.go @@ -0,0 +1,94 @@ +package restapi + +import ( + "net/http" + "strings" + + "maglev.onebusaway.org/internal/models" + "maglev.onebusaway.org/internal/utils" +) + +func (api *RestAPI) routeSearchHandler(w http.ResponseWriter, r *http.Request) { + queryParams := r.URL.Query() + + input := queryParams.Get("input") + sanitizedInput, err := utils.ValidateAndSanitizeQuery(input) + if err != nil { + fieldErrors := map[string][]string{ + "input": {err.Error()}, + } + api.validationErrorResponse(w, r, fieldErrors) + return + } + + if strings.TrimSpace(sanitizedInput) == "" { + fieldErrors := map[string][]string{ + "input": {"input is required"}, + } + api.validationErrorResponse(w, r, fieldErrors) + return + } + + maxCount := 20 + var fieldErrors map[string][]string + if maxCountStr := queryParams.Get("maxCount"); maxCountStr != "" { + parsedMaxCount, fe := utils.ParseFloatParam(queryParams, "maxCount", fieldErrors) + fieldErrors = fe + if parsedMaxCount <= 0 { + fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must be greater than zero") + } else { + maxCount = int(parsedMaxCount) + if maxCount > 100 { + fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must not exceed 100") + } + } + } + + if len(fieldErrors) > 0 { + api.validationErrorResponse(w, r, fieldErrors) + return + } + + ctx := r.Context() + if ctx.Err() != nil { + api.serverErrorResponse(w, r, ctx.Err()) + return + } + + routes, err := api.GtfsManager.SearchRoutes(ctx, sanitizedInput, maxCount) + if err != nil { + api.serverErrorResponse(w, r, err) + return + } + + results := make([]models.Route, 0, len(routes)) + agencyIDs := make(map[string]bool) + for _, routeRow := range routes { + agencyIDs[routeRow.AgencyID] = true + results = append(results, models.NewRoute( + utils.FormCombinedID(routeRow.AgencyID, routeRow.ID), + routeRow.AgencyID, + routeRow.ShortName.String, + routeRow.LongName.String, + routeRow.Desc.String, + models.RouteType(routeRow.Type), + routeRow.Url.String, + routeRow.Color.String, + routeRow.TextColor.String, + routeRow.ShortName.String, + )) + } + + agencies := utils.FilterAgencies(api.GtfsManager.GetAgencies(), agencyIDs) + references := models.ReferencesModel{ + Agencies: agencies, + Routes: []interface{}{}, + Situations: []interface{}{}, + StopTimes: []interface{}{}, + Stops: []models.Stop{}, + Trips: []interface{}{}, + } + + response := models.NewListResponse(results, references) + api.sendResponse(w, r, response) +} diff --git a/internal/restapi/route_search_handler_test.go b/internal/restapi/route_search_handler_test.go new file mode 100644 index 00000000..f10f0aca --- /dev/null +++ b/internal/restapi/route_search_handler_test.go @@ -0,0 +1,69 @@ +package restapi + +import ( + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRouteSearchHandlerRequiresValidApiKey(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=invalid&input=1") + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) + assert.Equal(t, http.StatusUnauthorized, model.Code) + assert.Equal(t, "permission denied", model.Text) +} + +func TestRouteSearchHandlerEndToEnd(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=shasta") + + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, http.StatusOK, model.Code) + assert.Equal(t, "OK", model.Text) + + data, ok := model.Data.(map[string]interface{}) + require.True(t, ok) + + list, ok := data["list"].([]interface{}) + require.True(t, ok) + assert.NotEmpty(t, list) + + found := false + for _, item := range list { + route, ok := item.(map[string]interface{}) + require.True(t, ok) + assert.Contains(t, route, "id") + assert.Contains(t, route, "agencyId") + assert.Contains(t, route, "shortName") + assert.Contains(t, route, "longName") + assert.Contains(t, route, "type") + + if route["shortName"] == "17" { + longName, _ := route["longName"].(string) + assert.True(t, strings.Contains(strings.ToLower(longName), "shasta")) + found = true + } + } + assert.True(t, found, "expected Shasta route to be returned") + + refs, ok := data["references"].(map[string]interface{}) + require.True(t, ok) + + agencies, ok := refs["agencies"].([]interface{}) + require.True(t, ok) + assert.NotEmpty(t, agencies) +} + +func TestRouteSearchHandlerRequiresInput(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + assert.Zero(t, model.Code) +} + +func TestRouteSearchHandlerValidatesMaxCount(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=1&maxCount=-1") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + assert.Zero(t, model.Code) +} diff --git a/internal/restapi/routes.go b/internal/restapi/routes.go index be94f074..a3cca0b0 100644 --- a/internal/restapi/routes.go +++ b/internal/restapi/routes.go @@ -75,6 +75,7 @@ func (api *RestAPI) SetRoutes(mux *http.ServeMux) { mux.Handle("GET /api/where/arrival-and-departure-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalAndDepartureForStopHandler)) mux.Handle("GET /api/where/trips-for-route/{id}", rateLimitAndValidateAPIKey(api, api.tripsForRouteHandler)) mux.Handle("GET /api/where/arrivals-and-departures-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalsAndDeparturesForStopHandler)) + mux.Handle("GET /api/where/search/route.json", rateLimitAndValidateAPIKey(api, api.routeSearchHandler)) } // SetupAPIRoutes creates and configures the API router with all middleware applied globally diff --git a/internal/utils/filters_test.go b/internal/utils/filters_test.go index db1520dc..4206b67b 100644 --- a/internal/utils/filters_test.go +++ b/internal/utils/filters_test.go @@ -11,7 +11,7 @@ import ( "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/models" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestFilterAgencies(t *testing.T) { From c8c72c0b95fae348c68a195684f33118be496ae3 Mon Sep 17 00:00:00 2001 From: Bhupesh Kumar Date: Thu, 11 Dec 2025 05:51:21 +0530 Subject: [PATCH 2/5] Revert "fixed" This reverts commit 8110f8cda53b143050b2b6e08c9c042e7a16c59c. --- cmd/api/app_test.go | 3 +- go.mod | 12 ++- go.sum | 44 +++++---- gtfsdb/bulk_insert_test.go | 2 +- gtfsdb/client.go | 2 +- gtfsdb/conditional_import_test.go | 2 +- gtfsdb/connection_pool_test.go | 4 +- gtfsdb/db.go | 10 -- gtfsdb/error_handling_test.go | 2 +- gtfsdb/helpers.go | 4 +- gtfsdb/nil_shape_test.go | 2 +- gtfsdb/query.sql | 25 ----- gtfsdb/query.sql.go | 66 ------------- gtfsdb/schema.sql | 72 -------------- gtfsdb/sqlite_performance_test.go | 4 +- internal/gtfs/gtfs_manager.go | 2 +- internal/gtfs/route_search.go | 46 --------- internal/restapi/route_search_handler.go | 94 ------------------- internal/restapi/route_search_handler_test.go | 69 -------------- internal/restapi/routes.go | 1 - internal/utils/filters_test.go | 2 +- 21 files changed, 51 insertions(+), 417 deletions(-) delete mode 100644 internal/gtfs/route_search.go delete mode 100644 internal/restapi/route_search_handler.go delete mode 100644 internal/restapi/route_search_handler_test.go diff --git a/cmd/api/app_test.go b/cmd/api/app_test.go index 8ea2893d..2db15ae9 100644 --- a/cmd/api/app_test.go +++ b/cmd/api/app_test.go @@ -9,11 +9,11 @@ import ( "testing" "time" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/gtfs" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestParseAPIKeys(t *testing.T) { @@ -445,7 +445,6 @@ func TestBuildApplicationWithConfigFile(t *testing.T) { // Convert to absolute path to avoid path traversal validation issues absTestDataPath, err := filepath.Abs(testDataPath) require.NoError(t, err) - absTestDataPath = filepath.ToSlash(absTestDataPath) // Create a test config file that uses the test data testConfigPath := filepath.Join("..", "..", "testdata", "config_test_build.json") diff --git a/go.mod b/go.mod index 42897ed1..a09d4aa0 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/OneBusAway/go-gtfs v1.1.0 github.com/davecgh/go-spew v1.1.1 github.com/klauspost/compress v1.18.0 + github.com/mattn/go-sqlite3 v1.14.24 github.com/stretchr/testify v1.10.0 github.com/tidwall/rtree v1.10.0 github.com/twpayne/go-polyline v1.1.1 golang.org/x/time v0.12.0 - modernc.org/sqlite v1.40.1 ) require ( @@ -31,6 +31,10 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 // indirect + github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 // indirect + github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 // indirect + github.com/paulmach/go.geojson v1.5.0 // indirect github.com/pganalyze/pg_query_go/v6 v6.1.0 // indirect github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect @@ -45,6 +49,7 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/tidwall/geoindex v1.7.0 // indirect + github.com/valyala/fastjson v1.6.4 // indirect github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect go.uber.org/atomic v1.11.0 // indirect @@ -54,7 +59,7 @@ require ( golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.36.0 // indirect + golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect @@ -62,9 +67,10 @@ require ( google.golang.org/protobuf v1.36.6 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.66.10 // indirect + modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.37.0 // indirect ) tool github.com/sqlc-dev/sqlc/cmd/sqlc diff --git a/go.sum b/go.sum index 676d8d17..78d46807 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,18 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1 h1:ei2LAhpj7frAPBzbjqTA9ICXi7H2KhBXlYzO0WwP8hI= +github.com/patrickbr/gtfsparser v0.0.0-20250811204933-790d4e1c69c1/go.mod h1:WsjXLsxSQc+KfBJ7APQhk3yz+4DztzMYARQ+EfxKYSQ= +github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868 h1:5RbZdJNYJbywbCwaQOzHUckzJ+F0w54imKZhkL0eUB0= +github.com/patrickbr/gtfstidy v0.0.0-20251119090422-df802af33868/go.mod h1:qNytqg0F2xv/NCYHECvfjr/QPioK94q2rhPTC7NI4KA= +github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8 h1:xEQhft28izewbL3fnF10Yj8zSNu2r+lW157GQq+eaaY= +github.com/patrickbr/gtfswriter v0.0.0-20240919073412-98e3602c6cd8/go.mod h1:zYAfZRXtDhc4Tq+LYrvyksefggNiViL2pKMRUrGvUbE= +github.com/paulmach/go.geojson v1.5.0 h1:7mhpMK89SQdHFcEGomT7/LuJhwhEgfmpWYVlVmLEdQw= +github.com/paulmach/go.geojson v1.5.0/go.mod h1:DgdUy2rRVDDVgKqrjMe2vZAHMfhDTrjVKt3LmHIXGbU= github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls= github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -111,6 +121,8 @@ github.com/tidwall/rtree v1.10.0 h1:+EcI8fboEaW1L3/9oW/6AMoQ8HiEIHyR7bQOGnmz4Mg= github.com/tidwall/rtree v1.10.0/go.mod h1:iDJQ9NBRtbfKkzZu02za+mIlaP+bjYPnunbSNidpbCQ= github.com/twpayne/go-polyline v1.1.1 h1:/tSF1BR7rN4HWj4XKqvRUNrCiYVMCvywxTFVofvDV0w= github.com/twpayne/go-polyline v1.1.1/go.mod h1:ybd9IWWivW/rlXPXuuckeKUyF3yrIim+iqA7kSl4NFY= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfPe7z7go8Dvv1AJQDI3eQ/5xith3q2mFlo= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07/go.mod h1:Ak17IJ037caFp4jpCw/iQQ7/W74Sqpb1YuKJU6HTKfM= github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= @@ -148,8 +160,8 @@ golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZv golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc= golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= @@ -159,8 +171,8 @@ golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= @@ -169,8 +181,8 @@ golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= @@ -196,18 +208,18 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= -modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= -modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= -modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= -modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= +modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= +modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= -modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -216,8 +228,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY= -modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= +modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= +modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/gtfsdb/bulk_insert_test.go b/gtfsdb/bulk_insert_test.go index f8688de1..3b6b3ee6 100644 --- a/gtfsdb/bulk_insert_test.go +++ b/gtfsdb/bulk_insert_test.go @@ -6,10 +6,10 @@ import ( "testing" "time" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestBulkInsertStopTimes(t *testing.T) { diff --git a/gtfsdb/client.go b/gtfsdb/client.go index f22156b4..bc269bd7 100644 --- a/gtfsdb/client.go +++ b/gtfsdb/client.go @@ -10,7 +10,7 @@ import ( "os" "time" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver ) // Client is the main entry point for the library diff --git a/gtfsdb/conditional_import_test.go b/gtfsdb/conditional_import_test.go index 321dfaf4..2462b58b 100644 --- a/gtfsdb/conditional_import_test.go +++ b/gtfsdb/conditional_import_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) // getTestFixturePath returns the absolute path to a fixture file in the testdata directory diff --git a/gtfsdb/connection_pool_test.go b/gtfsdb/connection_pool_test.go index e2491294..9784848e 100644 --- a/gtfsdb/connection_pool_test.go +++ b/gtfsdb/connection_pool_test.go @@ -5,10 +5,10 @@ import ( "database/sql" "testing" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestDatabaseConnectionPoolSettings(t *testing.T) { @@ -103,7 +103,7 @@ func TestConnectionLifetime(t *testing.T) { func TestConnectionPoolConfiguration(t *testing.T) { // Test the specific configuration values for in-memory databases - db, err := sql.Open("sqlite", ":memory:") + db, err := sql.Open("sqlite3", ":memory:") require.NoError(t, err, "Should open database") defer func() { _ = db.Close() }() diff --git a/gtfsdb/db.go b/gtfsdb/db.go index f178d067..bb938e08 100644 --- a/gtfsdb/db.go +++ b/gtfsdb/db.go @@ -258,9 +258,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.listRoutesStmt, err = db.PrepareContext(ctx, listRoutes); err != nil { return nil, fmt.Errorf("error preparing query ListRoutes: %w", err) } - if q.searchRoutesByFullTextStmt, err = db.PrepareContext(ctx, searchRoutesByFullText); err != nil { - return nil, fmt.Errorf("error preparing query SearchRoutesByFullText: %w", err) - } if q.listStopsStmt, err = db.PrepareContext(ctx, listStops); err != nil { return nil, fmt.Errorf("error preparing query ListStops: %w", err) } @@ -668,11 +665,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing listRoutesStmt: %w", cerr) } } - if q.searchRoutesByFullTextStmt != nil { - if cerr := q.searchRoutesByFullTextStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing searchRoutesByFullTextStmt: %w", cerr) - } - } if q.listStopsStmt != nil { if cerr := q.listStopsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing listStopsStmt: %w", cerr) @@ -810,7 +802,6 @@ type Queries struct { getTripsInBlockStmt *sql.Stmt listAgenciesStmt *sql.Stmt listRoutesStmt *sql.Stmt - searchRoutesByFullTextStmt *sql.Stmt listStopsStmt *sql.Stmt listTripsStmt *sql.Stmt updateStopDirectionStmt *sql.Stmt @@ -899,7 +890,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTripsInBlockStmt: q.getTripsInBlockStmt, listAgenciesStmt: q.listAgenciesStmt, listRoutesStmt: q.listRoutesStmt, - searchRoutesByFullTextStmt: q.searchRoutesByFullTextStmt, listStopsStmt: q.listStopsStmt, listTripsStmt: q.listTripsStmt, updateStopDirectionStmt: q.updateStopDirectionStmt, diff --git a/gtfsdb/error_handling_test.go b/gtfsdb/error_handling_test.go index 66d790ed..ff732966 100644 --- a/gtfsdb/error_handling_test.go +++ b/gtfsdb/error_handling_test.go @@ -6,10 +6,10 @@ import ( "net/http/httptest" "testing" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestNewClient_InvalidConfigHandling(t *testing.T) { diff --git a/gtfsdb/helpers.go b/gtfsdb/helpers.go index 7259ee15..bb6b329a 100644 --- a/gtfsdb/helpers.go +++ b/gtfsdb/helpers.go @@ -15,9 +15,9 @@ import ( "time" "github.com/OneBusAway/go-gtfs" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/logging" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) //go:embed schema.sql @@ -29,7 +29,7 @@ func createDB(config Config) (*sql.DB, error) { return nil, fmt.Errorf("test database must use in-memory storage, got path: %s", config.DBPath) } - db, err := sql.Open("sqlite", config.DBPath) + db, err := sql.Open("sqlite3", config.DBPath) if err != nil { return nil, err } diff --git a/gtfsdb/nil_shape_test.go b/gtfsdb/nil_shape_test.go index 6163b711..d322a2a2 100644 --- a/gtfsdb/nil_shape_test.go +++ b/gtfsdb/nil_shape_test.go @@ -6,9 +6,9 @@ import ( "context" "testing" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) // createMinimalGTFSWithoutShapes creates a minimal GTFS zip file that has trips WITHOUT shape_id diff --git a/gtfsdb/query.sql b/gtfsdb/query.sql index 7fbfe57f..c10cf71d 100644 --- a/gtfsdb/query.sql +++ b/gtfsdb/query.sql @@ -152,31 +152,6 @@ ORDER BY agency_id, id; --- name: SearchRoutesByFullText :many -SELECT - r.id, - r.agency_id, - r.short_name, - r.long_name, - r."desc", - r.type, - r.url, - r.color, - r.text_color, - r.continuous_pickup, - r.continuous_drop_off -FROM - routes_fts - JOIN routes r ON r.rowid = routes_fts.rowid -WHERE - routes_fts MATCH @query -ORDER BY - bm25(routes_fts), - r.agency_id, - r.id -LIMIT - @limit; - -- name: GetRouteIDsForAgency :many SELECT r.id diff --git a/gtfsdb/query.sql.go b/gtfsdb/query.sql.go index 053e7d3c..3b3d490c 100644 --- a/gtfsdb/query.sql.go +++ b/gtfsdb/query.sql.go @@ -3487,72 +3487,6 @@ func (q *Queries) ListRoutes(ctx context.Context) ([]Route, error) { return items, nil } -const searchRoutesByFullText = `-- name: SearchRoutesByFullText :many -SELECT - r.id, - r.agency_id, - r.short_name, - r.long_name, - r."desc", - r.type, - r.url, - r.color, - r.text_color, - r.continuous_pickup, - r.continuous_drop_off -FROM - routes_fts - JOIN routes r ON r.rowid = routes_fts.rowid -WHERE - routes_fts MATCH ? -ORDER BY - bm25(routes_fts), - r.agency_id, - r.id -LIMIT - ? -` - -type SearchRoutesByFullTextParams struct { - Query string - Limit int64 -} - -func (q *Queries) SearchRoutesByFullText(ctx context.Context, arg SearchRoutesByFullTextParams) ([]Route, error) { - rows, err := q.query(ctx, q.searchRoutesByFullTextStmt, searchRoutesByFullText, arg.Query, arg.Limit) - if err != nil { - return nil, err - } - defer rows.Close() - var items []Route - for rows.Next() { - var i Route - if err := rows.Scan( - &i.ID, - &i.AgencyID, - &i.ShortName, - &i.LongName, - &i.Desc, - &i.Type, - &i.Url, - &i.Color, - &i.TextColor, - &i.ContinuousPickup, - &i.ContinuousDropOff, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Close(); err != nil { - return nil, err - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const listStops = `-- name: ListStops :many SELECT id, code, name, "desc", lat, lon, zone_id, url, location_type, timezone, wheelchair_boarding, platform_code, direction diff --git a/gtfsdb/schema.sql b/gtfsdb/schema.sql index 91bc75b9..e0713c60 100644 --- a/gtfsdb/schema.sql +++ b/gtfsdb/schema.sql @@ -30,78 +30,6 @@ CREATE TABLE FOREIGN KEY (agency_id) REFERENCES agencies (id) ); --- migrate -CREATE VIRTUAL TABLE IF NOT EXISTS routes_fts USING fts5 ( - id UNINDEXED, - agency_id UNINDEXED, - short_name, - long_name, - desc, - content = 'routes', - content_rowid = 'rowid' -); - --- migrate -CREATE TRIGGER IF NOT EXISTS routes_fts_ai AFTER INSERT ON routes BEGIN -INSERT INTO - routes_fts(rowid, id, agency_id, short_name, long_name, desc) -VALUES - ( - new.rowid, - new.id, - new.agency_id, - coalesce(new.short_name, ''), - coalesce(new.long_name, ''), - coalesce(new.desc, '') - ); -END; - --- migrate -CREATE TRIGGER IF NOT EXISTS routes_fts_ad AFTER DELETE ON routes BEGIN -INSERT INTO - routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) -VALUES - ( - 'delete', - old.rowid, - old.id, - old.agency_id, - coalesce(old.short_name, ''), - coalesce(old.long_name, ''), - coalesce(old.desc, '') - ); -END; - --- migrate -CREATE TRIGGER IF NOT EXISTS routes_fts_au AFTER UPDATE ON routes BEGIN -INSERT INTO - routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) -VALUES - ( - 'delete', - old.rowid, - old.id, - old.agency_id, - coalesce(old.short_name, ''), - coalesce(old.long_name, ''), - coalesce(old.desc, '') - ); -INSERT INTO - routes_fts(rowid, id, agency_id, short_name, long_name, desc) -VALUES - ( - new.rowid, - new.id, - new.agency_id, - coalesce(new.short_name, ''), - coalesce(new.long_name, ''), - coalesce(new.desc, '') - ); -END; - --- migrate -INSERT INTO routes_fts(routes_fts) VALUES ('rebuild'); - -- migrate CREATE TABLE IF NOT EXISTS stops ( diff --git a/gtfsdb/sqlite_performance_test.go b/gtfsdb/sqlite_performance_test.go index 005c0fd1..41ff4529 100644 --- a/gtfsdb/sqlite_performance_test.go +++ b/gtfsdb/sqlite_performance_test.go @@ -7,10 +7,10 @@ import ( "path/filepath" "testing" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "maglev.onebusaway.org/internal/appconf" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) func TestSQLitePerformancePragmasApplied(t *testing.T) { @@ -218,7 +218,7 @@ func TestConfigureConnectionPoolWithDifferentConfigs(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - db, err := sql.Open("sqlite", ":memory:") + db, err := sql.Open("sqlite3", ":memory:") require.NoError(t, err) defer func() { _ = db.Close() }() diff --git a/internal/gtfs/gtfs_manager.go b/internal/gtfs/gtfs_manager.go index 4ad08e80..a05f417c 100644 --- a/internal/gtfs/gtfs_manager.go +++ b/internal/gtfs/gtfs_manager.go @@ -13,8 +13,8 @@ import ( "maglev.onebusaway.org/internal/utils" "github.com/OneBusAway/go-gtfs" + _ "github.com/mattn/go-sqlite3" // CGo-based SQLite driver "github.com/tidwall/rtree" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support ) const NoRadiusLimit = -1 diff --git a/internal/gtfs/route_search.go b/internal/gtfs/route_search.go deleted file mode 100644 index fbbae9f3..00000000 --- a/internal/gtfs/route_search.go +++ /dev/null @@ -1,46 +0,0 @@ -package gtfs - -import ( - "context" - "strings" - - "maglev.onebusaway.org/gtfsdb" -) - -// buildRouteSearchQuery normalizes user input into an FTS5-safe prefix search query. -func buildRouteSearchQuery(input string) string { - terms := strings.Fields(strings.ToLower(input)) - safeTerms := make([]string, 0, len(terms)) - - for _, term := range terms { - trimmed := strings.TrimSpace(term) - if trimmed == "" { - continue - } - escaped := strings.ReplaceAll(trimmed, `"`, `""`) - safeTerms = append(safeTerms, `"`+escaped+`"*`) - } - - if len(safeTerms) == 0 { - return "" - } - - return strings.Join(safeTerms, " AND ") -} - -// SearchRoutes performs a full text search against routes using SQLite FTS5. -func (manager *Manager) SearchRoutes(ctx context.Context, input string, maxCount int) ([]gtfsdb.Route, error) { - limit := maxCount - if limit <= 0 { - limit = 20 - } - if limit > 100 { - limit = 100 - } - - query := buildRouteSearchQuery(input) - return manager.GtfsDB.Queries.SearchRoutesByFullText(ctx, gtfsdb.SearchRoutesByFullTextParams{ - Query: query, - Limit: int64(limit), - }) -} diff --git a/internal/restapi/route_search_handler.go b/internal/restapi/route_search_handler.go deleted file mode 100644 index 35177d68..00000000 --- a/internal/restapi/route_search_handler.go +++ /dev/null @@ -1,94 +0,0 @@ -package restapi - -import ( - "net/http" - "strings" - - "maglev.onebusaway.org/internal/models" - "maglev.onebusaway.org/internal/utils" -) - -func (api *RestAPI) routeSearchHandler(w http.ResponseWriter, r *http.Request) { - queryParams := r.URL.Query() - - input := queryParams.Get("input") - sanitizedInput, err := utils.ValidateAndSanitizeQuery(input) - if err != nil { - fieldErrors := map[string][]string{ - "input": {err.Error()}, - } - api.validationErrorResponse(w, r, fieldErrors) - return - } - - if strings.TrimSpace(sanitizedInput) == "" { - fieldErrors := map[string][]string{ - "input": {"input is required"}, - } - api.validationErrorResponse(w, r, fieldErrors) - return - } - - maxCount := 20 - var fieldErrors map[string][]string - if maxCountStr := queryParams.Get("maxCount"); maxCountStr != "" { - parsedMaxCount, fe := utils.ParseFloatParam(queryParams, "maxCount", fieldErrors) - fieldErrors = fe - if parsedMaxCount <= 0 { - fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must be greater than zero") - } else { - maxCount = int(parsedMaxCount) - if maxCount > 100 { - fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must not exceed 100") - } - } - } - - if len(fieldErrors) > 0 { - api.validationErrorResponse(w, r, fieldErrors) - return - } - - ctx := r.Context() - if ctx.Err() != nil { - api.serverErrorResponse(w, r, ctx.Err()) - return - } - - routes, err := api.GtfsManager.SearchRoutes(ctx, sanitizedInput, maxCount) - if err != nil { - api.serverErrorResponse(w, r, err) - return - } - - results := make([]models.Route, 0, len(routes)) - agencyIDs := make(map[string]bool) - for _, routeRow := range routes { - agencyIDs[routeRow.AgencyID] = true - results = append(results, models.NewRoute( - utils.FormCombinedID(routeRow.AgencyID, routeRow.ID), - routeRow.AgencyID, - routeRow.ShortName.String, - routeRow.LongName.String, - routeRow.Desc.String, - models.RouteType(routeRow.Type), - routeRow.Url.String, - routeRow.Color.String, - routeRow.TextColor.String, - routeRow.ShortName.String, - )) - } - - agencies := utils.FilterAgencies(api.GtfsManager.GetAgencies(), agencyIDs) - references := models.ReferencesModel{ - Agencies: agencies, - Routes: []interface{}{}, - Situations: []interface{}{}, - StopTimes: []interface{}{}, - Stops: []models.Stop{}, - Trips: []interface{}{}, - } - - response := models.NewListResponse(results, references) - api.sendResponse(w, r, response) -} diff --git a/internal/restapi/route_search_handler_test.go b/internal/restapi/route_search_handler_test.go deleted file mode 100644 index f10f0aca..00000000 --- a/internal/restapi/route_search_handler_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package restapi - -import ( - "net/http" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRouteSearchHandlerRequiresValidApiKey(t *testing.T) { - _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=invalid&input=1") - assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) - assert.Equal(t, http.StatusUnauthorized, model.Code) - assert.Equal(t, "permission denied", model.Text) -} - -func TestRouteSearchHandlerEndToEnd(t *testing.T) { - _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=shasta") - - assert.Equal(t, http.StatusOK, resp.StatusCode) - assert.Equal(t, http.StatusOK, model.Code) - assert.Equal(t, "OK", model.Text) - - data, ok := model.Data.(map[string]interface{}) - require.True(t, ok) - - list, ok := data["list"].([]interface{}) - require.True(t, ok) - assert.NotEmpty(t, list) - - found := false - for _, item := range list { - route, ok := item.(map[string]interface{}) - require.True(t, ok) - assert.Contains(t, route, "id") - assert.Contains(t, route, "agencyId") - assert.Contains(t, route, "shortName") - assert.Contains(t, route, "longName") - assert.Contains(t, route, "type") - - if route["shortName"] == "17" { - longName, _ := route["longName"].(string) - assert.True(t, strings.Contains(strings.ToLower(longName), "shasta")) - found = true - } - } - assert.True(t, found, "expected Shasta route to be returned") - - refs, ok := data["references"].(map[string]interface{}) - require.True(t, ok) - - agencies, ok := refs["agencies"].([]interface{}) - require.True(t, ok) - assert.NotEmpty(t, agencies) -} - -func TestRouteSearchHandlerRequiresInput(t *testing.T) { - _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=") - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - assert.Zero(t, model.Code) -} - -func TestRouteSearchHandlerValidatesMaxCount(t *testing.T) { - _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=1&maxCount=-1") - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - assert.Zero(t, model.Code) -} diff --git a/internal/restapi/routes.go b/internal/restapi/routes.go index a3cca0b0..be94f074 100644 --- a/internal/restapi/routes.go +++ b/internal/restapi/routes.go @@ -75,7 +75,6 @@ func (api *RestAPI) SetRoutes(mux *http.ServeMux) { mux.Handle("GET /api/where/arrival-and-departure-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalAndDepartureForStopHandler)) mux.Handle("GET /api/where/trips-for-route/{id}", rateLimitAndValidateAPIKey(api, api.tripsForRouteHandler)) mux.Handle("GET /api/where/arrivals-and-departures-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalsAndDeparturesForStopHandler)) - mux.Handle("GET /api/where/search/route.json", rateLimitAndValidateAPIKey(api, api.routeSearchHandler)) } // SetupAPIRoutes creates and configures the API router with all middleware applied globally diff --git a/internal/utils/filters_test.go b/internal/utils/filters_test.go index 4206b67b..db1520dc 100644 --- a/internal/utils/filters_test.go +++ b/internal/utils/filters_test.go @@ -11,7 +11,7 @@ import ( "maglev.onebusaway.org/internal/appconf" "maglev.onebusaway.org/internal/models" - _ "modernc.org/sqlite" // Pure Go SQLite driver with FTS5 support + _ "github.com/mattn/go-sqlite3" ) func TestFilterAgencies(t *testing.T) { From 866c104e3a843ed7ea1f76c0cec35c40d207bfea Mon Sep 17 00:00:00 2001 From: Bhupesh Kumar Date: Thu, 11 Dec 2025 05:56:05 +0530 Subject: [PATCH 3/5] Refactor route search to sqlite3 + FTS5 tag --- README.markdown | 12 +++ cmd/api/app_test.go | 1 + gtfsdb/db.go | 10 ++ gtfsdb/query.sql | 25 +++++ gtfsdb/query.sql.go | 66 +++++++++++++ gtfsdb/schema.sql | 72 ++++++++++++++ internal/gtfs/route_search.go | 46 +++++++++ internal/restapi/route_search_handler.go | 94 +++++++++++++++++++ internal/restapi/route_search_handler_test.go | 67 +++++++++++++ internal/restapi/routes.go | 1 + 10 files changed, 394 insertions(+) create mode 100644 internal/gtfs/route_search.go create mode 100644 internal/restapi/route_search_handler.go create mode 100644 internal/restapi/route_search_handler_test.go diff --git a/README.markdown b/README.markdown index af333a82..9922ae6d 100644 --- a/README.markdown +++ b/README.markdown @@ -119,6 +119,18 @@ All basic commands are managed by our Makefile: `make watch` - Build and run the app with Air for live reloading during development (automatically rebuilds and restarts on code changes). +### FTS5 (SQLite) builds and tests + +The server uses `github.com/mattn/go-sqlite3` and SQLite FTS5 for route search. Build and test with the FTS5 tag enabled: + +```bash +CGO_ENABLED=1 go test -tags "sqlite_fts5" ./... +# or +CGO_ENABLED=1 go build -tags "sqlite_fts5" ./... +``` + +Ensure you have a working C toolchain when CGO is enabled. + ## Directory Structure * `bin` contains compiled application binaries, ready for deployment to a production server. diff --git a/cmd/api/app_test.go b/cmd/api/app_test.go index 2db15ae9..65a666b1 100644 --- a/cmd/api/app_test.go +++ b/cmd/api/app_test.go @@ -445,6 +445,7 @@ func TestBuildApplicationWithConfigFile(t *testing.T) { // Convert to absolute path to avoid path traversal validation issues absTestDataPath, err := filepath.Abs(testDataPath) require.NoError(t, err) + absTestDataPath = filepath.ToSlash(absTestDataPath) // Create a test config file that uses the test data testConfigPath := filepath.Join("..", "..", "testdata", "config_test_build.json") diff --git a/gtfsdb/db.go b/gtfsdb/db.go index bb938e08..f178d067 100644 --- a/gtfsdb/db.go +++ b/gtfsdb/db.go @@ -258,6 +258,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.listRoutesStmt, err = db.PrepareContext(ctx, listRoutes); err != nil { return nil, fmt.Errorf("error preparing query ListRoutes: %w", err) } + if q.searchRoutesByFullTextStmt, err = db.PrepareContext(ctx, searchRoutesByFullText); err != nil { + return nil, fmt.Errorf("error preparing query SearchRoutesByFullText: %w", err) + } if q.listStopsStmt, err = db.PrepareContext(ctx, listStops); err != nil { return nil, fmt.Errorf("error preparing query ListStops: %w", err) } @@ -665,6 +668,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing listRoutesStmt: %w", cerr) } } + if q.searchRoutesByFullTextStmt != nil { + if cerr := q.searchRoutesByFullTextStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchRoutesByFullTextStmt: %w", cerr) + } + } if q.listStopsStmt != nil { if cerr := q.listStopsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing listStopsStmt: %w", cerr) @@ -802,6 +810,7 @@ type Queries struct { getTripsInBlockStmt *sql.Stmt listAgenciesStmt *sql.Stmt listRoutesStmt *sql.Stmt + searchRoutesByFullTextStmt *sql.Stmt listStopsStmt *sql.Stmt listTripsStmt *sql.Stmt updateStopDirectionStmt *sql.Stmt @@ -890,6 +899,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTripsInBlockStmt: q.getTripsInBlockStmt, listAgenciesStmt: q.listAgenciesStmt, listRoutesStmt: q.listRoutesStmt, + searchRoutesByFullTextStmt: q.searchRoutesByFullTextStmt, listStopsStmt: q.listStopsStmt, listTripsStmt: q.listTripsStmt, updateStopDirectionStmt: q.updateStopDirectionStmt, diff --git a/gtfsdb/query.sql b/gtfsdb/query.sql index c10cf71d..7fbfe57f 100644 --- a/gtfsdb/query.sql +++ b/gtfsdb/query.sql @@ -152,6 +152,31 @@ ORDER BY agency_id, id; +-- name: SearchRoutesByFullText :many +SELECT + r.id, + r.agency_id, + r.short_name, + r.long_name, + r."desc", + r.type, + r.url, + r.color, + r.text_color, + r.continuous_pickup, + r.continuous_drop_off +FROM + routes_fts + JOIN routes r ON r.rowid = routes_fts.rowid +WHERE + routes_fts MATCH @query +ORDER BY + bm25(routes_fts), + r.agency_id, + r.id +LIMIT + @limit; + -- name: GetRouteIDsForAgency :many SELECT r.id diff --git a/gtfsdb/query.sql.go b/gtfsdb/query.sql.go index 3b3d490c..053e7d3c 100644 --- a/gtfsdb/query.sql.go +++ b/gtfsdb/query.sql.go @@ -3487,6 +3487,72 @@ func (q *Queries) ListRoutes(ctx context.Context) ([]Route, error) { return items, nil } +const searchRoutesByFullText = `-- name: SearchRoutesByFullText :many +SELECT + r.id, + r.agency_id, + r.short_name, + r.long_name, + r."desc", + r.type, + r.url, + r.color, + r.text_color, + r.continuous_pickup, + r.continuous_drop_off +FROM + routes_fts + JOIN routes r ON r.rowid = routes_fts.rowid +WHERE + routes_fts MATCH ? +ORDER BY + bm25(routes_fts), + r.agency_id, + r.id +LIMIT + ? +` + +type SearchRoutesByFullTextParams struct { + Query string + Limit int64 +} + +func (q *Queries) SearchRoutesByFullText(ctx context.Context, arg SearchRoutesByFullTextParams) ([]Route, error) { + rows, err := q.query(ctx, q.searchRoutesByFullTextStmt, searchRoutesByFullText, arg.Query, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Route + for rows.Next() { + var i Route + if err := rows.Scan( + &i.ID, + &i.AgencyID, + &i.ShortName, + &i.LongName, + &i.Desc, + &i.Type, + &i.Url, + &i.Color, + &i.TextColor, + &i.ContinuousPickup, + &i.ContinuousDropOff, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listStops = `-- name: ListStops :many SELECT id, code, name, "desc", lat, lon, zone_id, url, location_type, timezone, wheelchair_boarding, platform_code, direction diff --git a/gtfsdb/schema.sql b/gtfsdb/schema.sql index e0713c60..91bc75b9 100644 --- a/gtfsdb/schema.sql +++ b/gtfsdb/schema.sql @@ -30,6 +30,78 @@ CREATE TABLE FOREIGN KEY (agency_id) REFERENCES agencies (id) ); +-- migrate +CREATE VIRTUAL TABLE IF NOT EXISTS routes_fts USING fts5 ( + id UNINDEXED, + agency_id UNINDEXED, + short_name, + long_name, + desc, + content = 'routes', + content_rowid = 'rowid' +); + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_ai AFTER INSERT ON routes BEGIN +INSERT INTO + routes_fts(rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + new.rowid, + new.id, + new.agency_id, + coalesce(new.short_name, ''), + coalesce(new.long_name, ''), + coalesce(new.desc, '') + ); +END; + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_ad AFTER DELETE ON routes BEGIN +INSERT INTO + routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + 'delete', + old.rowid, + old.id, + old.agency_id, + coalesce(old.short_name, ''), + coalesce(old.long_name, ''), + coalesce(old.desc, '') + ); +END; + +-- migrate +CREATE TRIGGER IF NOT EXISTS routes_fts_au AFTER UPDATE ON routes BEGIN +INSERT INTO + routes_fts(routes_fts, rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + 'delete', + old.rowid, + old.id, + old.agency_id, + coalesce(old.short_name, ''), + coalesce(old.long_name, ''), + coalesce(old.desc, '') + ); +INSERT INTO + routes_fts(rowid, id, agency_id, short_name, long_name, desc) +VALUES + ( + new.rowid, + new.id, + new.agency_id, + coalesce(new.short_name, ''), + coalesce(new.long_name, ''), + coalesce(new.desc, '') + ); +END; + +-- migrate +INSERT INTO routes_fts(routes_fts) VALUES ('rebuild'); + -- migrate CREATE TABLE IF NOT EXISTS stops ( diff --git a/internal/gtfs/route_search.go b/internal/gtfs/route_search.go new file mode 100644 index 00000000..fbbae9f3 --- /dev/null +++ b/internal/gtfs/route_search.go @@ -0,0 +1,46 @@ +package gtfs + +import ( + "context" + "strings" + + "maglev.onebusaway.org/gtfsdb" +) + +// buildRouteSearchQuery normalizes user input into an FTS5-safe prefix search query. +func buildRouteSearchQuery(input string) string { + terms := strings.Fields(strings.ToLower(input)) + safeTerms := make([]string, 0, len(terms)) + + for _, term := range terms { + trimmed := strings.TrimSpace(term) + if trimmed == "" { + continue + } + escaped := strings.ReplaceAll(trimmed, `"`, `""`) + safeTerms = append(safeTerms, `"`+escaped+`"*`) + } + + if len(safeTerms) == 0 { + return "" + } + + return strings.Join(safeTerms, " AND ") +} + +// SearchRoutes performs a full text search against routes using SQLite FTS5. +func (manager *Manager) SearchRoutes(ctx context.Context, input string, maxCount int) ([]gtfsdb.Route, error) { + limit := maxCount + if limit <= 0 { + limit = 20 + } + if limit > 100 { + limit = 100 + } + + query := buildRouteSearchQuery(input) + return manager.GtfsDB.Queries.SearchRoutesByFullText(ctx, gtfsdb.SearchRoutesByFullTextParams{ + Query: query, + Limit: int64(limit), + }) +} diff --git a/internal/restapi/route_search_handler.go b/internal/restapi/route_search_handler.go new file mode 100644 index 00000000..35177d68 --- /dev/null +++ b/internal/restapi/route_search_handler.go @@ -0,0 +1,94 @@ +package restapi + +import ( + "net/http" + "strings" + + "maglev.onebusaway.org/internal/models" + "maglev.onebusaway.org/internal/utils" +) + +func (api *RestAPI) routeSearchHandler(w http.ResponseWriter, r *http.Request) { + queryParams := r.URL.Query() + + input := queryParams.Get("input") + sanitizedInput, err := utils.ValidateAndSanitizeQuery(input) + if err != nil { + fieldErrors := map[string][]string{ + "input": {err.Error()}, + } + api.validationErrorResponse(w, r, fieldErrors) + return + } + + if strings.TrimSpace(sanitizedInput) == "" { + fieldErrors := map[string][]string{ + "input": {"input is required"}, + } + api.validationErrorResponse(w, r, fieldErrors) + return + } + + maxCount := 20 + var fieldErrors map[string][]string + if maxCountStr := queryParams.Get("maxCount"); maxCountStr != "" { + parsedMaxCount, fe := utils.ParseFloatParam(queryParams, "maxCount", fieldErrors) + fieldErrors = fe + if parsedMaxCount <= 0 { + fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must be greater than zero") + } else { + maxCount = int(parsedMaxCount) + if maxCount > 100 { + fieldErrors["maxCount"] = append(fieldErrors["maxCount"], "must not exceed 100") + } + } + } + + if len(fieldErrors) > 0 { + api.validationErrorResponse(w, r, fieldErrors) + return + } + + ctx := r.Context() + if ctx.Err() != nil { + api.serverErrorResponse(w, r, ctx.Err()) + return + } + + routes, err := api.GtfsManager.SearchRoutes(ctx, sanitizedInput, maxCount) + if err != nil { + api.serverErrorResponse(w, r, err) + return + } + + results := make([]models.Route, 0, len(routes)) + agencyIDs := make(map[string]bool) + for _, routeRow := range routes { + agencyIDs[routeRow.AgencyID] = true + results = append(results, models.NewRoute( + utils.FormCombinedID(routeRow.AgencyID, routeRow.ID), + routeRow.AgencyID, + routeRow.ShortName.String, + routeRow.LongName.String, + routeRow.Desc.String, + models.RouteType(routeRow.Type), + routeRow.Url.String, + routeRow.Color.String, + routeRow.TextColor.String, + routeRow.ShortName.String, + )) + } + + agencies := utils.FilterAgencies(api.GtfsManager.GetAgencies(), agencyIDs) + references := models.ReferencesModel{ + Agencies: agencies, + Routes: []interface{}{}, + Situations: []interface{}{}, + StopTimes: []interface{}{}, + Stops: []models.Stop{}, + Trips: []interface{}{}, + } + + response := models.NewListResponse(results, references) + api.sendResponse(w, r, response) +} diff --git a/internal/restapi/route_search_handler_test.go b/internal/restapi/route_search_handler_test.go new file mode 100644 index 00000000..eb0857c8 --- /dev/null +++ b/internal/restapi/route_search_handler_test.go @@ -0,0 +1,67 @@ +package restapi + +import ( + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRouteSearchHandlerRequiresValidApiKey(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=invalid&input=1") + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) + assert.Equal(t, http.StatusUnauthorized, model.Code) + assert.Equal(t, "permission denied", model.Text) +} + +func TestRouteSearchHandlerEndToEnd(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=shasta") + + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, http.StatusOK, model.Code) + assert.Equal(t, "OK", model.Text) + + data, ok := model.Data.(map[string]interface{}) + require.True(t, ok) + + list, ok := data["list"].([]interface{}) + require.True(t, ok) + assert.NotEmpty(t, list) + + found := false + for _, item := range list { + route, ok := item.(map[string]interface{}) + require.True(t, ok) + assert.Contains(t, route, "id") + assert.Contains(t, route, "agencyId") + assert.Contains(t, route, "shortName") + assert.Contains(t, route, "longName") + assert.Contains(t, route, "type") + + if route["shortName"] == "17" { + longName, _ := route["longName"].(string) + assert.True(t, strings.Contains(strings.ToLower(longName), "shasta")) + found = true + } + } + assert.True(t, found, "expected Shasta route to be returned") + + refs, ok := data["references"].(map[string]interface{}) + require.True(t, ok) + + agencies, ok := refs["agencies"].([]interface{}) + require.True(t, ok) + assert.NotEmpty(t, agencies) +} + +func TestRouteSearchHandlerRequiresInput(t *testing.T) { + _, resp, _ := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestRouteSearchHandlerValidatesMaxCount(t *testing.T) { + _, resp, _ := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=1&maxCount=-1") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} diff --git a/internal/restapi/routes.go b/internal/restapi/routes.go index be94f074..a3cca0b0 100644 --- a/internal/restapi/routes.go +++ b/internal/restapi/routes.go @@ -75,6 +75,7 @@ func (api *RestAPI) SetRoutes(mux *http.ServeMux) { mux.Handle("GET /api/where/arrival-and-departure-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalAndDepartureForStopHandler)) mux.Handle("GET /api/where/trips-for-route/{id}", rateLimitAndValidateAPIKey(api, api.tripsForRouteHandler)) mux.Handle("GET /api/where/arrivals-and-departures-for-stop/{id}", rateLimitAndValidateAPIKey(api, api.arrivalsAndDeparturesForStopHandler)) + mux.Handle("GET /api/where/search/route.json", rateLimitAndValidateAPIKey(api, api.routeSearchHandler)) } // SetupAPIRoutes creates and configures the API router with all middleware applied globally From 24fb2a17bace525e68371d9fd4f6cd3f722871fa Mon Sep 17 00:00:00 2001 From: Bhup-GitHUB Date: Wed, 21 Jan 2026 15:26:49 +0530 Subject: [PATCH 4/5] ci: Enable FTS5 support in GitHub Actions workflow --- .github/workflows/go.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 8f197ce2..ca434930 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -25,7 +25,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: latest - args: --timeout=5m + args: --timeout=5m --build-tags=fts5 test: name: Test @@ -46,7 +46,9 @@ jobs: run: go mod verify - name: Run tests - run: go test -v -coverprofile=profile.cov ./... + run: go test -v -tags=fts5 -coverprofile=profile.cov ./... + env: + CGO_ENABLED: 1 # Disabling coverage reporting for now. Re-enable once the project is made public. # - uses: shogo82148/actions-goveralls@v1 From 20219009bff2957b1b992017bf9bd32673865fa3 Mon Sep 17 00:00:00 2001 From: Bhup-GitHUB Date: Tue, 27 Jan 2026 00:12:42 +0530 Subject: [PATCH 5/5] Fix route search issues per maintainer feedback --- gtfsdb/schema.sql | 4 ++ internal/gtfs/route_search.go | 18 +++++++-- internal/restapi/rate_limit_shutdown_test.go | 3 +- internal/restapi/route_search_handler.go | 40 +++++++++++++++---- internal/restapi/route_search_handler_test.go | 30 ++++++++++++++ 5 files changed, 82 insertions(+), 13 deletions(-) diff --git a/gtfsdb/schema.sql b/gtfsdb/schema.sql index 91bc75b9..eebe1218 100644 --- a/gtfsdb/schema.sql +++ b/gtfsdb/schema.sql @@ -31,6 +31,9 @@ CREATE TABLE ); -- migrate +-- FTS5 external content table for full-text route search. +-- Data lives in 'routes' table; only the search index is stored here. +-- The triggers below keep the index synchronized with the content table. CREATE VIRTUAL TABLE IF NOT EXISTS routes_fts USING fts5 ( id UNINDEXED, agency_id UNINDEXED, @@ -42,6 +45,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS routes_fts USING fts5 ( ); -- migrate +-- Trigger naming: ai=After Insert, ad=After Delete, au=After Update CREATE TRIGGER IF NOT EXISTS routes_fts_ai AFTER INSERT ON routes BEGIN INSERT INTO routes_fts(rowid, id, agency_id, short_name, long_name, desc) diff --git a/internal/gtfs/route_search.go b/internal/gtfs/route_search.go index fbbae9f3..513d3987 100644 --- a/internal/gtfs/route_search.go +++ b/internal/gtfs/route_search.go @@ -2,6 +2,8 @@ package gtfs import ( "context" + "fmt" + "log/slog" "strings" "maglev.onebusaway.org/gtfsdb" @@ -34,13 +36,21 @@ func (manager *Manager) SearchRoutes(ctx context.Context, input string, maxCount if limit <= 0 { limit = 20 } - if limit > 100 { - limit = 100 - } query := buildRouteSearchQuery(input) - return manager.GtfsDB.Queries.SearchRoutesByFullText(ctx, gtfsdb.SearchRoutesByFullTextParams{ + if query == "" { + return []gtfsdb.Route{}, nil + } + + logger := slog.Default().With(slog.String("component", "route_search")) + logger.Debug("route search", slog.String("input", input), slog.String("query", query), slog.Int("limit", limit)) + + routes, err := manager.GtfsDB.Queries.SearchRoutesByFullText(ctx, gtfsdb.SearchRoutesByFullTextParams{ Query: query, Limit: int64(limit), }) + if err != nil { + return nil, fmt.Errorf("route search failed for query %q: %w", query, err) + } + return routes, nil } diff --git a/internal/restapi/rate_limit_shutdown_test.go b/internal/restapi/rate_limit_shutdown_test.go index 066a530d..9e862e5b 100644 --- a/internal/restapi/rate_limit_shutdown_test.go +++ b/internal/restapi/rate_limit_shutdown_test.go @@ -64,7 +64,7 @@ func TestRateLimitMiddleware_GoroutineActuallyExits(t *testing.T) { // Force garbage collection to clean up any lingering goroutines from previous tests runtime.GC() time.Sleep(50 * time.Millisecond) - + // Get baseline goroutine count initial := runtime.NumGoroutine() @@ -80,4 +80,3 @@ func TestRateLimitMiddleware_GoroutineActuallyExits(t *testing.T) { afterStop := runtime.NumGoroutine() assert.LessOrEqual(t, afterStop, initial, "cleanup goroutine should have exited") } - diff --git a/internal/restapi/route_search_handler.go b/internal/restapi/route_search_handler.go index 35177d68..726003b7 100644 --- a/internal/restapi/route_search_handler.go +++ b/internal/restapi/route_search_handler.go @@ -65,17 +65,43 @@ func (api *RestAPI) routeSearchHandler(w http.ResponseWriter, r *http.Request) { agencyIDs := make(map[string]bool) for _, routeRow := range routes { agencyIDs[routeRow.AgencyID] = true + + shortName := "" + if routeRow.ShortName.Valid { + shortName = routeRow.ShortName.String + } + longName := "" + if routeRow.LongName.Valid { + longName = routeRow.LongName.String + } + desc := "" + if routeRow.Desc.Valid { + desc = routeRow.Desc.String + } + url := "" + if routeRow.Url.Valid { + url = routeRow.Url.String + } + color := "" + if routeRow.Color.Valid { + color = routeRow.Color.String + } + textColor := "" + if routeRow.TextColor.Valid { + textColor = routeRow.TextColor.String + } + results = append(results, models.NewRoute( utils.FormCombinedID(routeRow.AgencyID, routeRow.ID), routeRow.AgencyID, - routeRow.ShortName.String, - routeRow.LongName.String, - routeRow.Desc.String, + shortName, + longName, + desc, models.RouteType(routeRow.Type), - routeRow.Url.String, - routeRow.Color.String, - routeRow.TextColor.String, - routeRow.ShortName.String, + url, + color, + textColor, + shortName, )) } diff --git a/internal/restapi/route_search_handler_test.go b/internal/restapi/route_search_handler_test.go index eb0857c8..553b90aa 100644 --- a/internal/restapi/route_search_handler_test.go +++ b/internal/restapi/route_search_handler_test.go @@ -65,3 +65,33 @@ func TestRouteSearchHandlerValidatesMaxCount(t *testing.T) { _, resp, _ := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=1&maxCount=-1") assert.Equal(t, http.StatusBadRequest, resp.StatusCode) } + +func TestRouteSearchHandlerNoResults(t *testing.T) { + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=zzzznonexistent99999") + + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, http.StatusOK, model.Code) + + data, ok := model.Data.(map[string]interface{}) + require.True(t, ok) + + list, ok := data["list"].([]interface{}) + require.True(t, ok) + assert.Empty(t, list) +} + +func TestRouteSearchHandlerWhitespaceInput(t *testing.T) { + _, resp, _ := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=%20%20%20") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +} + +func TestRouteSearchHandlerMaxCountBoundaries(t *testing.T) { + // Exactly 100 should work + _, resp, model := serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=shasta&maxCount=100") + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, http.StatusOK, model.Code) + + // 101 should fail + _, resp, _ = serveAndRetrieveEndpoint(t, "/api/where/search/route.json?key=TEST&input=shasta&maxCount=101") + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) +}