p
/
gallery
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: p:f9f22ba42c1f72540fd28576eb568142da7cd03c
Branches Tags
p:master
...
compare: p:5e0e9f8a42d3b68e744abcd95f10d6a52f866463
Branches Tags
p:master

3 Commits
f9f22ba42c ... 5e0e9f8a42

Author SHA1 Message Date
Přemysl Eric Janouch 5e0e9f8a42
gallery: clean up, search in a transaction 2024-01-22 19:52:37 +01:00
Přemysl Eric Janouch 083739fd4e
gallery: implement AND/NOT for tag search 2024-01-22 19:52:35 +01:00
Přemysl Eric Janouch 4f174972e3
gallery: move out a query from CTE 2024-01-22 19:51:13 +01:00
4 changed files with 179 additions and 81 deletions
Show Stats Expand all files Collapse all files

2
initialize.sql
View File

@ -76,7 +76,7 @@ CREATE TABLE IF NOT EXISTS tag_space(
id INTEGER NOT NULL,
name TEXT NOT NULL,
description TEXT,
CHECK (name NOT LIKE '%:%'),
CHECK (name NOT LIKE '%:%' AND name NOT LIKE '-%'),
PRIMARY KEY (id)
) STRICT;

250
main.go
View File

@ -62,10 +62,33 @@ func hammingDistance(a, b int64) int {
return bits.OnesCount64(uint64(a) ^ uint64(b))
}
type productAggregator float64
func (pa *productAggregator) Step(v float64) {
*pa = productAggregator(float64(*pa) * v)
}
func (pa *productAggregator) Done() float64 {
return float64(*pa)
}
func newProductAggregator() *productAggregator {
pa := productAggregator(1)
return &pa
}
func init() {
sql.Register("sqlite3_custom", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("hamming", hammingDistance, true /*pure*/)
if err := conn.RegisterFunc(
"hamming", hammingDistance, true /*pure*/); err != nil {
return err
}
if err := conn.RegisterAggregator(
"product", newProductAggregator, true /*pure*/); err != nil {
return err
}
return nil
},
})
}
@ -294,49 +317,7 @@ func cmdInit(fs *flag.FlagSet, args []string) error {
return nil
}
// --- Web ---------------------------------------------------------------------
var hashRE = regexp.MustCompile(`^/.*?/([0-9a-f]{40})$`)
var staticHandler http.Handler
var page = template.Must(template.New("/").Parse(`<!DOCTYPE html><html><head>
<title>Gallery</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel=stylesheet href=style.css>
</head><body>
<noscript>This is a web application, and requires Javascript.</noscript>
<script src=mithril.js></script>
<script src=gallery.js></script>
</body></html>`))
func handleRequest(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
staticHandler.ServeHTTP(w, r)
return
}
if err := page.Execute(w, nil); err != nil {
log.Println(err)
}
}
func handleImages(w http.ResponseWriter, r *http.Request) {
if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
http.NotFound(w, r)
} else {
http.ServeFile(w, r, imagePath(m[1]))
}
}
func handleThumbs(w http.ResponseWriter, r *http.Request) {
if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
http.NotFound(w, r)
} else {
http.ServeFile(w, r, thumbPath(m[1]))
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Browse -------------------------------------------------------------
func getSubdirectories(tx *sql.Tx, parent int64) (names []string, err error) {
return dbCollectStrings(`SELECT name FROM node
@ -416,7 +397,7 @@ func handleAPIBrowse(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Tags ---------------------------------------------------------------
type webTagNamespace struct {
Description string `json:"description"`
@ -502,7 +483,7 @@ func handleAPITags(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Duplicates ---------------------------------------------------------
type webDuplicateImage struct {
SHA1 string `json:"sha1"`
@ -645,7 +626,7 @@ func handleAPIDuplicates(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Orphans ------------------------------------------------------------
type webOrphanImage struct {
SHA1 string `json:"sha1"`
@ -742,7 +723,7 @@ func handleAPIOrphans(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Image view ---------------------------------------------------------
func getImageDimensions(sha1 string) (w int64, h int64, err error) {
err = db.QueryRow(`SELECT width, height FROM image WHERE sha1 = ?`,
@ -845,7 +826,7 @@ func handleAPIInfo(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Image similar ------------------------------------------------------
type webSimilarImage struct {
SHA1 string `json:"sha1"`
@ -955,25 +936,90 @@ func handleAPISimilar(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- API: Search -------------------------------------------------------------
// The SQL building is the most miserable part of the whole program.
// NOTE: AND will mean MULTIPLY(IFNULL(ta.weight, 0)) per SHA1.
const searchCTE = `WITH
const searchCTE1 = `WITH
matches(sha1, thumbw, thumbh, score) AS (
SELECT i.sha1, i.thumbw, i.thumbh, ta.weight AS score
FROM tag_assignment AS ta
JOIN image AS i ON i.sha1 = ta.sha1
WHERE ta.tag = ?
),
supertags(tag, space, name) AS (
SELECT DISTINCT ta.tag, ts.name, t.name
FROM tag_assignment AS ta
JOIN matches AS m ON m.sha1 = ta.sha1
JOIN tag AS t ON ta.tag = t.id
JOIN tag_space AS ts ON ts.id = t.space
WHERE ta.tag = %d
)
`
const searchCTEMulti = `WITH
positive(tag) AS (VALUES %s),
filtered(sha1) AS (%s),
matches(sha1, thumbw, thumbh, score) AS (
SELECT i.sha1, i.thumbw, i.thumbh,
product(IFNULL(ta.weight, 0)) AS score
FROM image AS i, positive AS p
JOIN filtered AS c ON i.sha1 = c.sha1
LEFT JOIN tag_assignment AS ta ON ta.sha1 = i.sha1 AND ta.tag = p.tag
GROUP BY i.sha1
)
`
func searchQueryToCTE(tx *sql.Tx, query string) (string, error) {
positive, negative := []int64{}, []int64{}
for _, word := range strings.Split(query, " ") {
if word == "" {
continue
}
space, tag, _ := strings.Cut(word, ":")
negated := false
if strings.HasPrefix(space, "-") {
space = space[1:]
negated = true
}
var tagID int64
err := tx.QueryRow(`
SELECT t.id FROM tag AS t
JOIN tag_space AS ts ON t.space = ts.id
WHERE ts.name = ? AND t.name = ?`, space, tag).Scan(&tagID)
if err != nil {
return "", err
}
if negated {
negative = append(negative, tagID)
} else {
positive = append(positive, tagID)
}
}
// Don't return most of the database, and simplify the following builder.
if len(positive) == 0 {
return "", errors.New("search is too wide")
}
// Optimise single tag searches.
if len(positive) == 1 && len(negative) == 0 {
return fmt.Sprintf(searchCTE1, positive[0]), nil
}
values := fmt.Sprintf(`(%d)`, positive[0])
filtered := fmt.Sprintf(
`SELECT sha1 FROM tag_assignment WHERE tag = %d`, positive[0])
for _, tagID := range positive[1:] {
values += fmt.Sprintf(`, (%d)`, tagID)
filtered += fmt.Sprintf(` INTERSECT
SELECT sha1 FROM tag_assignment WHERE tag = %d`, tagID)
}
for _, tagID := range negative {
filtered += fmt.Sprintf(` EXCEPT
SELECT sha1 FROM tag_assignment WHERE tag = %d`, tagID)
}
return fmt.Sprintf(searchCTEMulti, values, filtered), nil
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
type webTagMatch struct {
SHA1 string `json:"sha1"`
ThumbW int64 `json:"thumbW"`
@ -981,10 +1027,10 @@ type webTagMatch struct {
Score float32 `json:"score"`
}
func getTagMatches(tag int64) (matches []webTagMatch, err error) {
rows, err := db.Query(searchCTE+`
func getTagMatches(tx *sql.Tx, cte string) (matches []webTagMatch, err error) {
rows, err := tx.Query(cte + `
SELECT sha1, IFNULL(thumbw, 0), IFNULL(thumbh, 0), score
FROM matches`, tag)
FROM matches`)
if err != nil {
return nil, err
}
@ -1008,9 +1054,14 @@ type webTagSupertag struct {
score float32
}
func getTagSupertags(tag int64) (result map[int64]*webTagSupertag, err error) {
rows, err := db.Query(searchCTE+`
SELECT tag, space, name FROM supertags`, tag)
func getTagSupertags(tx *sql.Tx, cte string) (
result map[int64]*webTagSupertag, err error) {
rows, err := tx.Query(cte + `
SELECT DISTINCT ta.tag, ts.name, t.name
FROM tag_assignment AS ta
JOIN matches AS m ON m.sha1 = ta.sha1
JOIN tag AS t ON ta.tag = t.id
JOIN tag_space AS ts ON ts.id = t.space`)
if err != nil {
return nil, err
}
@ -1035,18 +1086,18 @@ type webTagRelated struct {
Score float32 `json:"score"`
}
func getTagRelated(tag int64, matches int) (
func getTagRelated(tx *sql.Tx, cte string, matches int) (
result map[string][]webTagRelated, err error) {
// Not sure if this level of efficiency is achievable directly in SQL.
supertags, err := getTagSupertags(tag)
supertags, err := getTagSupertags(tx, cte)
if err != nil {
return nil, err
}
rows, err := db.Query(searchCTE+`
rows, err := tx.Query(cte + `
SELECT ta.tag, ta.weight
FROM tag_assignment AS ta
JOIN matches AS m ON m.sha1 = ta.sha1`, tag)
JOIN matches AS m ON m.sha1 = ta.sha1`)
if err != nil {
return nil, err
}
@ -1087,13 +1138,14 @@ func handleAPISearch(w http.ResponseWriter, r *http.Request) {
Related map[string][]webTagRelated `json:"related"`
}
space, tag, _ := strings.Cut(params.Query, ":")
tx, err := db.Begin()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer tx.Rollback()
var tagID int64
err := db.QueryRow(`
SELECT t.id FROM tag AS t
JOIN tag_space AS ts ON t.space = ts.id
WHERE ts.name = ? AND t.name = ?`, space, tag).Scan(&tagID)
cte, err := searchQueryToCTE(tx, params.Query)
if errors.Is(err, sql.ErrNoRows) {
http.Error(w, err.Error(), http.StatusNotFound)
return
@ -1102,11 +1154,11 @@ func handleAPISearch(w http.ResponseWriter, r *http.Request) {
return
}
if result.Matches, err = getTagMatches(tagID); err != nil {
if result.Matches, err = getTagMatches(tx, cte); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if result.Related, err = getTagRelated(tagID,
if result.Related, err = getTagRelated(tx, cte,
len(result.Matches)); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -1117,7 +1169,47 @@ func handleAPISearch(w http.ResponseWriter, r *http.Request) {
}
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// --- Web ---------------------------------------------------------------------
var hashRE = regexp.MustCompile(`^/.*?/([0-9a-f]{40})$`)
var staticHandler http.Handler
var page = template.Must(template.New("/").Parse(`<!DOCTYPE html><html><head>
<title>Gallery</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel=stylesheet href=style.css>
</head><body>
<noscript>This is a web application, and requires Javascript.</noscript>
<script src=mithril.js></script>
<script src=gallery.js></script>
</body></html>`))
func handleRequest(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
staticHandler.ServeHTTP(w, r)
return
}
if err := page.Execute(w, nil); err != nil {
log.Println(err)
}
}
func handleImages(w http.ResponseWriter, r *http.Request) {
if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
http.NotFound(w, r)
} else {
http.ServeFile(w, r, imagePath(m[1]))
}
}
func handleThumbs(w http.ResponseWriter, r *http.Request) {
if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil {
http.NotFound(w, r)
} else {
http.ServeFile(w, r, thumbPath(m[1]))
}
}
// cmdWeb runs a web UI against GD on ADDRESS.
func cmdWeb(fs *flag.FlagSet, args []string) error {

6
public/gallery.js
View File

@ -646,7 +646,11 @@ let Search = {
m(Header),
m('.body', {}, [
m('.sidebar', [
m('p', SearchModel.query),
m('input', {
value: SearchModel.query,
onchange: event => m.route.set(
`/search/:key`, {key: event.target.value}),
}),
m(SearchRelated),
]),
m(SearchView),

2
public/style.css
View File

@ -27,6 +27,8 @@ a { color: inherit; }
.sidebar { padding: .25rem .5rem; background: var(--shade-color);
border-right: 1px solid #ccc; overflow: auto;
min-width: 10rem; max-width: 20rem; flex-shrink: 0; }
.sidebar input { width: 100%; box-sizing: border-box; margin: .5rem 0;
font-size: inherit; }
.sidebar h2 { margin: 0.5em 0 0.25em 0; padding: 0; font-size: 1.2rem; }
.sidebar ul { margin: .5rem 0; padding: 0; }