commit 054078908a1e4c7429ea0f5a3a0605addfccc46c Author: Přemysl Eric Janouch Date: Fri Dec 8 02:16:04 2023 +0100 Initial commit diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d13ecd --- /dev/null +++ b/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2023, Přemysl Eric Janouch + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fe30c13 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.POSIX: +.SUFFIXES: + +outputs = gallery initialize.go public/mithril.js +all: $(outputs) + +gallery: main.go initialize.go + go build -tags "" -gcflags="all=-N -l" -o $@ +initialize.go: initialize.sql gen-initialize.sh + ./gen-initialize.sh initialize.sql > $@ +public/mithril.js: + curl -Lo $@ https://unpkg.com/mithril/mithril.js +clean: + rm -f $(outputs) diff --git a/README b/README new file mode 100644 index 0000000..03a34fe --- /dev/null +++ b/README @@ -0,0 +1,14 @@ +This is gallery software designed to maintain a shadow structure +of your filesystem, in which you can attach metadata to your media, +and query your collections in various ways. + +All media is content-addressed by its SHA-1 hash value, and at your option +also perceptually hashed. Duplicate search is an essential feature. + +Prerequisites: Go, ImageMagick, xdg-utils + +The gallery is designed for simplicity, and easy interoperability. +sqlite3, curl, jq, and the filesystem will take you a long way. + +The intended mode of use is running daily automated sync/thumbnail/dhash/tag +batches in a cron job, or from a system timer. See test.sh for usage hints. diff --git a/gen-initialize.sh b/gen-initialize.sh new file mode 100755 index 0000000..8d8cb55 --- /dev/null +++ b/gen-initialize.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e +gofmt < + Gallery + + + + + + + +`)) + +func handleRequest(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + staticHandler.ServeHTTP(w, r) + return + } + if err := page.Execute(w, nil); err != nil { + log.Println(err) + } +} + +func handleImages(w http.ResponseWriter, r *http.Request) { + if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil { + http.NotFound(w, r) + } else { + http.ServeFile(w, r, imagePath(m[1])) + } +} + +func handleThumbs(w http.ResponseWriter, r *http.Request) { + if m := hashRE.FindStringSubmatch(r.URL.Path); m == nil { + http.NotFound(w, r) + } else { + http.ServeFile(w, r, thumbPath(m[1])) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func getSubdirectories(tx *sql.Tx, parent int64) (names []string, err error) { + return dbCollectStrings(`SELECT name FROM node + WHERE IFNULL(parent, 0) = ? AND sha1 IS NULL`, parent) +} + +type webEntry struct { + SHA1 string `json:"sha1"` + Name string `json:"name"` + Modified int64 `json:"modified"` + ThumbW int64 `json:"thumbW"` + ThumbH int64 `json:"thumbH"` +} + +func getSubentries(tx *sql.Tx, parent int64) (entries []webEntry, err error) { + rows, err := tx.Query(` + SELECT i.sha1, n.name, n.mtime, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0) + FROM node AS n + JOIN image AS i ON n.sha1 = i.sha1 + WHERE n.parent = ?`, parent) + if err != nil { + return nil, err + } + defer rows.Close() + + entries = []webEntry{} + for rows.Next() { + var e webEntry + if err = rows.Scan( + &e.SHA1, &e.Name, &e.Modified, &e.ThumbW, &e.ThumbH); err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, rows.Err() +} + +func handleAPIBrowse(w http.ResponseWriter, r *http.Request) { + var params struct { + Path string + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var result struct { + Subdirectories []string `json:"subdirectories"` + Entries []webEntry `json:"entries"` + } + + tx, err := db.Begin() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer tx.Rollback() + + parent, err := idForDirectoryPath(tx, decodeWebPath(params.Path), false) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + result.Subdirectories, err = getSubdirectories(tx, parent) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + result.Entries, err = getSubentries(tx, parent) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +type webTagNamespace struct { + Description string `json:"description"` + Tags map[string]int64 `json:"tags"` +} + +func getTags(nsID int64) (result map[string]int64, err error) { + rows, err := db.Query(` + SELECT t.name, COUNT(ta.tag) AS count + FROM tag AS t + LEFT JOIN tag_assignment AS ta ON t.id = ta.tag + WHERE t.space = ? + GROUP BY t.id`, nsID) + if err != nil { + return + } + defer rows.Close() + + result = make(map[string]int64) + for rows.Next() { + var ( + name string + count int64 + ) + if err = rows.Scan(&name, &count); err != nil { + return + } + result[name] = count + } + return result, rows.Err() +} + +func getTagNamespaces(match *string) ( + result map[string]webTagNamespace, err error) { + var rows *sql.Rows + if match != nil { + rows, err = db.Query(`SELECT id, name, IFNULL(description, '') + FROM tag_space WHERE name = ?`, *match) + } else { + rows, err = db.Query(`SELECT id, name, IFNULL(description, '') + FROM tag_space`) + } + if err != nil { + return + } + defer rows.Close() + + result = make(map[string]webTagNamespace) + for rows.Next() { + var ( + id int64 + name string + ns webTagNamespace + ) + if err = rows.Scan(&id, &name, &ns.Description); err != nil { + return + } + if ns.Tags, err = getTags(id); err != nil { + return + } + result[name] = ns + } + return result, rows.Err() +} + +func handleAPITags(w http.ResponseWriter, r *http.Request) { + var params struct { + Namespace *string + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + result, err := getTagNamespaces(params.Namespace) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +type webDuplicateImage struct { + SHA1 string `json:"sha1"` + ThumbW int64 `json:"thumbW"` + ThumbH int64 `json:"thumbH"` + Occurences int64 `json:"occurences"` +} + +// A hamming distance of zero (direct dhash match) will be more than sufficient. +const duplicatesCTE = `WITH + duplicated(dhash, count) AS ( + SELECT dhash, COUNT(*) AS count FROM image + WHERE dhash IS NOT NULL + GROUP BY dhash HAVING count > 1 + ), + multipathed(sha1, count) AS ( + SELECT n.sha1, COUNT(*) AS count FROM node AS n + JOIN image AS i ON i.sha1 = n.sha1 + WHERE i.dhash IS NULL + OR i.dhash NOT IN (SELECT dhash FROM duplicated) + GROUP BY n.sha1 HAVING count > 1 + ) +` + +func getDuplicatesSimilar(stmt *sql.Stmt, dhash int64) ( + result []webDuplicateImage, err error) { + rows, err := stmt.Query(dhash) + if err != nil { + return nil, err + } + defer rows.Close() + + result = []webDuplicateImage{} + for rows.Next() { + var image webDuplicateImage + if err = rows.Scan(&image.SHA1, &image.ThumbW, &image.ThumbH, + &image.Occurences); err != nil { + return nil, err + } + result = append(result, image) + } + return result, rows.Err() +} + +func getDuplicates1(result [][]webDuplicateImage) ( + [][]webDuplicateImage, error) { + stmt, err := db.Prepare(` + SELECT i.sha1, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), + COUNT(*) AS occurences + FROM image AS i + JOIN node AS n ON n.sha1 = i.sha1 + WHERE i.dhash = ? + GROUP BY n.sha1`) + if err != nil { + return nil, err + } + defer stmt.Close() + + rows, err := db.Query(duplicatesCTE + `SELECT dhash FROM duplicated`) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var ( + group []webDuplicateImage + dhash int64 + ) + if err = rows.Scan(&dhash); err != nil { + return nil, err + } + if group, err = getDuplicatesSimilar(stmt, dhash); err != nil { + return nil, err + } + result = append(result, group) + } + return result, rows.Err() +} + +func getDuplicates2(result [][]webDuplicateImage) ( + [][]webDuplicateImage, error) { + stmt, err := db.Prepare(` + SELECT i.sha1, IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), + COUNT(*) AS occurences + FROM image AS i + JOIN node AS n ON n.sha1 = i.sha1 + WHERE i.sha1 = ? + GROUP BY n.sha1`) + if err != nil { + return nil, err + } + defer stmt.Close() + + rows, err := db.Query(duplicatesCTE + `SELECT sha1 FROM multipathed`) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var ( + image webDuplicateImage + sha1 string + ) + if err = rows.Scan(&sha1); err != nil { + return nil, err + } + if err := stmt.QueryRow(sha1).Scan(&image.SHA1, + &image.ThumbW, &image.ThumbH, &image.Occurences); err != nil { + return nil, err + } + result = append(result, []webDuplicateImage{image}) + } + return result, rows.Err() +} + +func handleAPIDuplicates(w http.ResponseWriter, r *http.Request) { + var params struct{} + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var ( + result = [][]webDuplicateImage{} + err error + ) + if result, err = getDuplicates1(result); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if result, err = getDuplicates2(result); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +type webOrphanImage struct { + SHA1 string `json:"sha1"` + ThumbW int64 `json:"thumbW"` + ThumbH int64 `json:"thumbH"` + Tags int64 `json:"tags"` +} + +type webOrphan struct { + webOrphanImage + LastPath string `json:"lastPath"` + Replacement *webOrphanImage `json:"replacement"` +} + +func getOrphanReplacement(webPath string) (*webOrphanImage, error) { + tx, err := db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + path := decodeWebPath(webPath) + if len(path) == 0 { + return nil, nil + } + + parent, err := idForDirectoryPath(tx, path[:len(path)-1], false) + if err != nil { + return nil, err + } + + var image webOrphanImage + err = db.QueryRow(`SELECT i.sha1, + IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), COUNT(ta.sha1) AS tags + FROM node AS n + JOIN image AS i ON n.sha1 = i.sha1 + LEFT JOIN tag_assignment AS ta ON n.sha1 = ta.sha1 + WHERE n.parent = ? AND n.name = ? + GROUP BY n.sha1`, parent, path[len(path)-1]).Scan( + &image.SHA1, &image.ThumbW, &image.ThumbH, &image.Tags) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } else if err != nil { + return nil, err + } + return &image, nil +} + +func getOrphans() (result []webOrphan, err error) { + rows, err := db.Query(`SELECT o.sha1, o.path, + IFNULL(i.thumbw, 0), IFNULL(i.thumbh, 0), COUNT(ta.sha1) AS tags + FROM orphan AS o + JOIN image AS i ON o.sha1 = i.sha1 + LEFT JOIN tag_assignment AS ta ON o.sha1 = ta.sha1 + GROUP BY o.sha1`) + if err != nil { + return nil, err + } + defer rows.Close() + + result = []webOrphan{} + for rows.Next() { + var orphan webOrphan + if err = rows.Scan(&orphan.SHA1, &orphan.LastPath, + &orphan.ThumbW, &orphan.ThumbH, &orphan.Tags); err != nil { + return nil, err + } + + orphan.Replacement, err = getOrphanReplacement(orphan.LastPath) + if err != nil { + return nil, err + } + + result = append(result, orphan) + } + return result, rows.Err() +} + +func handleAPIOrphans(w http.ResponseWriter, r *http.Request) { + var params struct{} + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + result, err := getOrphans() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func getImageDimensions(sha1 string) (w int64, h int64, err error) { + err = db.QueryRow(`SELECT width, height FROM image WHERE sha1 = ?`, + sha1).Scan(&w, &h) + return +} + +func getImagePaths(sha1 string) (paths []string, err error) { + rows, err := db.Query(`WITH RECURSIVE paths(parent, path) AS ( + SELECT parent, name AS path FROM node WHERE sha1 = ? + UNION ALL + SELECT n.parent, n.name || '/' || p.path + FROM node AS n JOIN paths AS p ON n.id = p.parent + ) SELECT path FROM paths WHERE parent IS NULL`, sha1) + if err != nil { + return nil, err + } + defer rows.Close() + + paths = []string{} + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, err + } + paths = append(paths, path) + } + return paths, rows.Err() +} + +func getImageTags(sha1 string) (map[string]map[string]float32, error) { + rows, err := db.Query(` + SELECT ts.name, t.name, ta.weight FROM tag_assignment AS ta + JOIN tag AS t ON t.id = ta.tag + JOIN tag_space AS ts ON ts.id = t.space + WHERE ta.sha1 = ?`, sha1) + if err != nil { + return nil, err + } + defer rows.Close() + + result := make(map[string]map[string]float32) + for rows.Next() { + var ( + space, tag string + weight float32 + ) + if err := rows.Scan(&space, &tag, &weight); err != nil { + return nil, err + } + + tags := result[space] + if tags == nil { + tags = make(map[string]float32) + result[space] = tags + } + tags[tag] = weight + } + return result, rows.Err() +} + +func handleAPIInfo(w http.ResponseWriter, r *http.Request) { + var params struct { + SHA1 string + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var result struct { + Width int64 `json:"width"` + Height int64 `json:"height"` + Paths []string `json:"paths"` + Tags map[string]map[string]float32 `json:"tags"` + } + + var err error + result.Width, result.Height, err = getImageDimensions(params.SHA1) + if errors.Is(err, sql.ErrNoRows) { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + result.Paths, err = getImagePaths(params.SHA1) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + result.Tags, err = getImageTags(params.SHA1) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +type webSimilarImage struct { + SHA1 string `json:"sha1"` + PixelsRatio float32 `json:"pixelsRatio"` + ThumbW int64 `json:"thumbW"` + ThumbH int64 `json:"thumbH"` + Paths []string `json:"paths"` +} + +func getSimilar(sha1 string, dhash int64, pixels int64, distance int) ( + result []webSimilarImage, err error) { + // For distance ∈ {0, 1}, this query is quite inefficient. + // In exchange, it's generic. + // + // If there's a dhash, there should also be thumbnail dimensions, + // so not bothering with IFNULL on them. + rows, err := db.Query(` + SELECT sha1, width * height, IFNULL(thumbw, 0), IFNULL(thumbh, 0) + FROM image WHERE sha1 <> ? AND dhash IS NOT NULL + AND hamming(dhash, ?) = ?`, sha1, dhash, distance) + if err != nil { + return nil, err + } + defer rows.Close() + + result = []webSimilarImage{} + for rows.Next() { + var ( + match webSimilarImage + matchPixels int64 + ) + if err = rows.Scan(&match.SHA1, + &matchPixels, &match.ThumbW, &match.ThumbH); err != nil { + return nil, err + } + if match.Paths, err = getImagePaths(match.SHA1); err != nil { + return nil, err + } + match.PixelsRatio = float32(matchPixels) / float32(pixels) + result = append(result, match) + } + return result, rows.Err() +} + +func getSimilarGroups(sha1 string, dhash int64, pixels int64, + output map[string][]webSimilarImage) error { + var err error + for distance := 0; distance <= 1; distance++ { + output[fmt.Sprintf("Perceptual distance %d", distance)], err = + getSimilar(sha1, dhash, pixels, distance) + if err != nil { + return err + } + } + return nil +} + +func handleAPISimilar(w http.ResponseWriter, r *http.Request) { + var params struct { + SHA1 string + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var result struct { + Info webSimilarImage `json:"info"` + Groups map[string][]webSimilarImage `json:"groups"` + } + + result.Info = webSimilarImage{SHA1: params.SHA1, PixelsRatio: 1} + if paths, err := getImagePaths(params.SHA1); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } else { + result.Info.Paths = paths + } + + var ( + width, height int64 + dhash sql.NullInt64 + ) + err := db.QueryRow(` + SELECT width, height, dhash, IFNULL(thumbw, 0), IFNULL(thumbh, 0) + FROM image WHERE sha1 = ?`, params.SHA1).Scan(&width, &height, &dhash, + &result.Info.ThumbW, &result.Info.ThumbH) + if errors.Is(err, sql.ErrNoRows) { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + result.Groups = make(map[string][]webSimilarImage) + if dhash.Valid { + if err := getSimilarGroups( + params.SHA1, dhash.Int64, width*height, result.Groups); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// NOTE: AND will mean MULTIPLY(IFNULL(ta.weight, 0)) per SHA1. +const searchCTE = `WITH + matches(sha1, thumbw, thumbh, score) AS ( + SELECT i.sha1, i.thumbw, i.thumbh, ta.weight AS score + FROM tag_assignment AS ta + JOIN image AS i ON i.sha1 = ta.sha1 + WHERE ta.tag = ? + ), + supertags(tag) AS ( + SELECT DISTINCT ta.tag + FROM tag_assignment AS ta + JOIN matches AS m ON m.sha1 = ta.sha1 + ), + scoredtags(tag, score) AS ( + -- The cross join is a deliberate optimization, + -- and this query may still be really slow. + SELECT st.tag, AVG(IFNULL(ta.weight, 0)) AS score + FROM matches AS m + CROSS JOIN supertags AS st + LEFT JOIN tag_assignment AS ta + ON ta.sha1 = m.sha1 AND ta.tag = st.tag + GROUP BY st.tag + -- Using the column alias doesn't fail, but it also doesn't work. + HAVING AVG(IFNULL(ta.weight, 0)) >= 0.01 + ) +` + +type webTagMatch struct { + SHA1 string `json:"sha1"` + ThumbW int64 `json:"thumbW"` + ThumbH int64 `json:"thumbH"` + Score float32 `json:"score"` +} + +func getTagMatches(tag int64) (matches []webTagMatch, err error) { + rows, err := db.Query(searchCTE+` + SELECT sha1, IFNULL(thumbw, 0), IFNULL(thumbh, 0), score + FROM matches`, tag) + if err != nil { + return nil, err + } + defer rows.Close() + + matches = []webTagMatch{} + for rows.Next() { + var match webTagMatch + if err = rows.Scan(&match.SHA1, + &match.ThumbW, &match.ThumbH, &match.Score); err != nil { + return nil, err + } + matches = append(matches, match) + } + return matches, rows.Err() +} + +type webTagRelated struct { + Tag string `json:"tag"` + Score float32 `json:"score"` +} + +func getTagRelated(tag int64) (result map[string][]webTagRelated, err error) { + rows, err := db.Query(searchCTE+` + SELECT ts.name, t.name, st.score FROM scoredtags AS st + JOIN tag AS t ON st.tag = t.id + JOIN tag_space AS ts ON ts.id = t.space + ORDER BY st.score DESC`, tag) + if err != nil { + return nil, err + } + defer rows.Close() + + result = make(map[string][]webTagRelated) + for rows.Next() { + var ( + space string + r webTagRelated + ) + if err = rows.Scan(&space, &r.Tag, &r.Score); err != nil { + return nil, err + } + result[space] = append(result[space], r) + } + return result, rows.Err() +} + +func handleAPISearch(w http.ResponseWriter, r *http.Request) { + var params struct { + Query string + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var result struct { + Matches []webTagMatch `json:"matches"` + Related map[string][]webTagRelated `json:"related"` + } + + space, tag, _ := strings.Cut(params.Query, ":") + + var tagID int64 + err := db.QueryRow(` + SELECT t.id FROM tag AS t + JOIN tag_space AS ts ON t.space = ts.id + WHERE ts.name = ? AND t.name = ?`, space, tag).Scan(&tagID) + if errors.Is(err, sql.ErrNoRows) { + http.Error(w, err.Error(), http.StatusNotFound) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if result.Matches, err = getTagMatches(tagID); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if result.Related, err = getTagRelated(tagID); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if err := json.NewEncoder(w).Encode(result); err != nil { + log.Println(err) + } +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// cmdWeb runs a web UI against GD on ADDRESS. +func cmdWeb(fs *flag.FlagSet, args []string) error { + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 2 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + address := fs.Arg(1) + + // This separation is not strictly necessary, + // but having an elementary level of security doesn't hurt either. + staticHandler = http.FileServer(http.Dir("public")) + + http.HandleFunc("/", handleRequest) + http.HandleFunc("/image/", handleImages) + http.HandleFunc("/thumb/", handleThumbs) + http.HandleFunc("/api/browse", handleAPIBrowse) + http.HandleFunc("/api/tags", handleAPITags) + http.HandleFunc("/api/duplicates", handleAPIDuplicates) + http.HandleFunc("/api/orphans", handleAPIOrphans) + http.HandleFunc("/api/info", handleAPIInfo) + http.HandleFunc("/api/similar", handleAPISimilar) + http.HandleFunc("/api/search", handleAPISearch) + + host, port, err := net.SplitHostPort(address) + if err != nil { + log.Println(err) + } else if host == "" { + log.Println("http://" + net.JoinHostPort("localhost", port)) + } else { + log.Println("http://" + address) + } + + s := &http.Server{ + Addr: address, + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, + MaxHeaderBytes: 32 << 10, + } + return s.ListenAndServe() +} + +// --- Sync -------------------------------------------------------------------- + +type syncFileInfo struct { + dbID int64 // DB node ID, or zero if there was none + dbParent int64 // where the file was to be stored + dbName string // the name under which it was to be stored + fsPath string // symlink target + fsMtime int64 // last modified Unix timestamp, used a bit like an ID + + err error // any processing error + sha1 string // raw content hash, empty to skip file + width int // image width in pixels + height int // image height in pixels +} + +type syncContext struct { + ctx context.Context + tx *sql.Tx + info chan syncFileInfo + pb *progressBar + + stmtOrphan *sql.Stmt + stmtDisposeSub *sql.Stmt + stmtDisposeAll *sql.Stmt + + // linked tracks which image hashes we've checked so far in the run. + linked map[string]struct{} +} + +func syncPrintf(c *syncContext, format string, v ...any) { + c.pb.Interrupt(func() { log.Printf(format+"\n", v...) }) +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +type syncNode struct { + dbID int64 + dbName string + dbMtime int64 + dbSHA1 string +} + +func (n *syncNode) dbIsDir() bool { return n.dbSHA1 == "" } + +type syncFile struct { + fsName string + fsMtime int64 + fsIsDir bool +} + +type syncPair struct { + db *syncNode + fs *syncFile +} + +// syncGetNodes returns direct children of a DB node, ordered by name. +// SQLite, like Go, compares strings byte-wise by default. +func syncGetNodes(tx *sql.Tx, dbParent int64) (nodes []syncNode, err error) { + // This works even for the root, which doesn't exist as a DB node. + rows, err := tx.Query(`SELECT id, name, IFNULL(mtime, 0), IFNULL(sha1, '') + FROM node WHERE IFNULL(parent, 0) = ? ORDER BY name`, dbParent) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var node syncNode + if err = rows.Scan(&node.dbID, + &node.dbName, &node.dbMtime, &node.dbSHA1); err != nil { + return + } + nodes = append(nodes, node) + } + return nodes, rows.Err() +} + +// syncGetFiles returns direct children of a FS directory, ordered by name. +func syncGetFiles(fsPath string) (files []syncFile, err error) { + dir, err := os.Open(fsPath) + if err != nil { + return + } + defer dir.Close() + + entries, err := dir.ReadDir(0) + if err != nil { + return + } + + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return files, err + } + + files = append(files, syncFile{ + fsName: entry.Name(), + fsMtime: info.ModTime().Unix(), + fsIsDir: entry.IsDir(), + }) + } + sort.Slice(files, + func(a, b int) bool { return files[a].fsName < files[b].fsName }) + return +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func syncIsImage(path string) (bool, error) { + out, err := exec.Command("xdg-mime", "query", "filetype", path).Output() + if err != nil { + return false, err + } + + return bytes.HasPrefix(out, []byte("image/")), nil +} + +func syncPingImage(path string) (int, int, error) { + out, err := exec.Command("magick", "identify", "-limit", "thread", "1", + "-ping", "-format", "%w %h", path+"[0]").Output() + if err != nil { + return 0, 0, err + } + + var w, h int + _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h) + return w, h, err +} + +func syncProcess(c *syncContext, info *syncFileInfo) error { + // Skip videos, which ImageMagick can process, but we don't want it to, + // so that they're not converted 1:1 to WebP. + pathIsImage, err := syncIsImage(info.fsPath) + if err != nil { + return err + } + if !pathIsImage { + return nil + } + + info.width, info.height, err = syncPingImage(info.fsPath) + if err != nil { + return err + } + + f, err := os.Open(info.fsPath) + if err != nil { + return err + } + defer f.Close() + + // We could make this at least somewhat interruptible by c.ctx, + // though it would still work poorly. + hash := sha1.New() + _, err = io.CopyBuffer(hash, f, make([]byte, 65536)) + if err != nil { + return err + } + + info.sha1 = hex.EncodeToString(hash.Sum(nil)) + return nil +} + +// syncEnqueue runs file scanning, which can be CPU and I/O expensive, +// in parallel. The goroutine only touches the filesystem, read-only. +func syncEnqueue(c *syncContext, info syncFileInfo) error { + if err := taskSemaphore.acquire(c.ctx); err != nil { + return err + } + + go func(info syncFileInfo) { + defer taskSemaphore.release() + info.err = syncProcess(c, &info) + c.info <- info + }(info) + return nil +} + +// syncDequeue flushes the result queue of finished asynchronous tasks. +func syncDequeue(c *syncContext) error { + for { + select { + case <-c.ctx.Done(): + return c.ctx.Err() + case info := <-c.info: + if err := syncPostProcess(c, info); err != nil { + return err + } + default: + return nil + } + } +} + +// syncDispose creates orphan records for the entire subtree given by nodeID +// as appropriate, then deletes all nodes within the subtree. The subtree root +// node is not deleted if "keepNode" is true. +// +// Orphans keep their thumbnail files, as evidence. +func syncDispose(c *syncContext, nodeID int64, keepNode bool) error { + if _, err := c.stmtOrphan.Exec(nodeID); err != nil { + return err + } + + if keepNode { + if _, err := c.stmtDisposeSub.Exec(nodeID); err != nil { + return err + } + } else { + if _, err := c.stmtDisposeAll.Exec(nodeID); err != nil { + return err + } + } + return nil +} + +func syncImageResave(c *syncContext, path string, target string) error { + dirname, _ := filepath.Split(path) + if err := os.MkdirAll(dirname, 0755); err != nil { + return err + } + + for { + // Try to remove anything standing in the way. + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + // TODO: Make it possible to copy or reflink (ioctl FICLONE). + err = os.Symlink(target, path) + if err == nil || !errors.Is(err, fs.ErrExist) { + return err + } + } +} + +func syncImageSave(c *syncContext, sha1 string, target string) error { + if _, ok := c.linked[sha1]; ok { + return nil + } + + ok, path := false, imagePath(sha1) + if link, err := os.Readlink(path); err == nil { + ok = link == target + } else { + // If it exists, but it is not a symlink, let it be. + // Even though it may not be a regular file. + ok = errors.Is(err, syscall.EINVAL) + } + + if !ok { + if err := syncImageResave(c, path, target); err != nil { + return err + } + } + + c.linked[sha1] = struct{}{} + return nil +} + +func syncImage(c *syncContext, info syncFileInfo) error { + if _, err := c.tx.Exec(`INSERT INTO image(sha1, width, height) + VALUES (?, ?, ?) ON CONFLICT(sha1) DO NOTHING`, + info.sha1, info.width, info.height); err != nil { + return err + } + + return syncImageSave(c, info.sha1, info.fsPath) +} + +func syncPostProcess(c *syncContext, info syncFileInfo) error { + defer c.pb.Step() + + // TODO: When replacing an image node (whether it has or doesn't have + // other links to keep it alive), we could offer copying all tags, + // though this needs another table to track it. + // (If it's equivalent enough, the dhash will stay the same, + // so user can resolve this through the duplicates feature.) + switch { + case info.err != nil: + // * → error + if ee, ok := info.err.(*exec.ExitError); ok { + syncPrintf(c, "%s: %s", info.fsPath, ee.Stderr) + } else { + return info.err + } + fallthrough + + case info.sha1 == "": + // 0 → 0 + if info.dbID == 0 { + return nil + } + + // D → 0, F → 0 + // TODO: Make it possible to disable removal (for copying only?) + return syncDispose(c, info.dbID, false /*keepNode*/) + + case info.dbID == 0: + // 0 → F + if err := syncImage(c, info); err != nil { + return err + } + if _, err := c.tx.Exec(`INSERT INTO node(parent, name, mtime, sha1) + VALUES (?, ?, ?, ?)`, + info.dbParent, info.dbName, info.fsMtime, info.sha1); err != nil { + return err + } + return nil + + default: + // D → F, F → F (this statement is a no-op with the latter) + if err := syncDispose(c, info.dbID, true /*keepNode*/); err != nil { + return err + } + + // Even if the hash didn't change, see comment in syncDirectoryPair(). + if err := syncImage(c, info); err != nil { + return err + } + if _, err := c.tx.Exec(`UPDATE node SET mtime = ?, sha1 = ? + WHERE id = ?`, info.fsMtime, info.sha1, info.dbID); err != nil { + return err + } + return nil + } +} + +func syncDirectoryPair(c *syncContext, dbParent int64, fsPath string, + pair syncPair) error { + db, fs, fsInfo := pair.db, pair.fs, syncFileInfo{dbParent: dbParent} + if db != nil { + fsInfo.dbID = db.dbID + } + if fs != nil { + fsInfo.dbName = fs.fsName + fsInfo.fsPath = filepath.Join(fsPath, fs.fsName) + fsInfo.fsMtime = fs.fsMtime + } + + switch { + case db == nil && fs == nil: + // 0 → 0, unreachable. + + case db == nil && fs.fsIsDir: + // 0 → D + var id int64 + if result, err := c.tx.Exec(`INSERT INTO node(parent, name) + VALUES (?, ?)`, dbParent, fs.fsName); err != nil { + return err + } else if id, err = result.LastInsertId(); err != nil { + return err + } + return syncDirectory(c, id, fsInfo.fsPath) + + case db == nil: + // 0 → F (or 0 → 0) + return syncEnqueue(c, fsInfo) + + case fs == nil: + // D → 0, F → 0 + // TODO: Make it possible to disable removal (for copying only?) + return syncDispose(c, db.dbID, false /*keepNode*/) + + case db.dbIsDir() && fs.fsIsDir: + // D → D + return syncDirectory(c, db.dbID, fsInfo.fsPath) + + case db.dbIsDir(): + // D → F (or D → 0) + return syncEnqueue(c, fsInfo) + + case fs.fsIsDir: + // F → D + if err := syncDispose(c, db.dbID, true /*keepNode*/); err != nil { + return err + } + if _, err := c.tx.Exec(`UPDATE node + SET mtime = NULL, sha1 = NULL WHERE id = ?`, db.dbID); err != nil { + return err + } + return syncDirectory(c, db.dbID, fsInfo.fsPath) + + case db.dbMtime != fs.fsMtime: + // F → F (or F → 0) + // Assuming that any content modifications change the timestamp. + return syncEnqueue(c, fsInfo) + + default: + // F → F + // Try to fix symlinks, to handle the following situations: + // 1. Image A occurs in paths 1 and 2, we use a symlink to path 1, + // and path 1 is removed from the filesystem: + // path 2 would not resolve if the mtime didn't change. + // 2. Image A occurs in paths 1 and 2, we use a symlink to path 1, + // and path 1 is changed: + // path 2 would resolve to the wrong file. + // This may relink images with multiple occurences unnecessarily, + // but it will always fix the roots that are being synced. + if err := syncImageSave(c, db.dbSHA1, fsInfo.fsPath); err != nil { + return err + } + } + return nil +} + +func syncDirectory(c *syncContext, dbParent int64, fsPath string) error { + db, err := syncGetNodes(c.tx, dbParent) + if err != nil { + return err + } + + fs, err := syncGetFiles(fsPath) + if err != nil { + return err + } + + // This would not be fatal, but it has annoying consequences. + if _, ok := slices.BinarySearchFunc(fs, syncFile{fsName: nameOfDB}, + func(a, b syncFile) int { + return strings.Compare(a.fsName, b.fsName) + }); ok { + syncPrintf(c, "%s may be a gallery directory, treating as empty", + fsPath) + fs = nil + } + + // Convert differences to a form more convenient for processing. + iDB, iFS, pairs := 0, 0, []syncPair{} + for iDB < len(db) && iFS < len(fs) { + if db[iDB].dbName == fs[iFS].fsName { + pairs = append(pairs, syncPair{&db[iDB], &fs[iFS]}) + iDB++ + iFS++ + } else if db[iDB].dbName < fs[iFS].fsName { + pairs = append(pairs, syncPair{&db[iDB], nil}) + iDB++ + } else { + pairs = append(pairs, syncPair{nil, &fs[iFS]}) + iFS++ + } + } + for i := range db[iDB:] { + pairs = append(pairs, syncPair{&db[iDB+i], nil}) + } + for i := range fs[iFS:] { + pairs = append(pairs, syncPair{nil, &fs[iFS+i]}) + } + + for _, pair := range pairs { + if err := syncDequeue(c); err != nil { + return err + } + if err := syncDirectoryPair(c, dbParent, fsPath, pair); err != nil { + return err + } + } + return nil +} + +func syncRoot(c *syncContext, dbPath []string, fsPath string) error { + // TODO: Support synchronizing individual files. + // This can only be treated as 0 → F, F → F, or D → F, that is, + // a variation on current syncEnqueue(), but dbParent must be nullable. + + // Figure out a database root (not trying to convert F → D on conflict, + // also because we don't know yet if the argument is a directory). + // + // Synchronizing F → D or * → F are special cases not worth implementing. + dbParent, err := idForDirectoryPath(c.tx, dbPath, true) + if err != nil { + return err + } + if err := syncDirectory(c, dbParent, fsPath); err != nil { + return err + } + + // Wait for all tasks to finish, and process the results of their work. + for i := 0; i < cap(taskSemaphore); i++ { + if err := taskSemaphore.acquire(c.ctx); err != nil { + return err + } + } + if err := syncDequeue(c); err != nil { + return err + } + + // This is not our semaphore, so prepare it for the next user. + for i := 0; i < cap(taskSemaphore); i++ { + taskSemaphore.release() + } + + // Delete empty directories, from the bottom of the tree up to, + // but not including, the inserted root. + // + // We need to do this at the end due to our recursive handling, + // as well as because of asynchronous file filtering. + stmt, err := c.tx.Prepare(` + WITH RECURSIVE subtree(id, parent, sha1, level) AS ( + SELECT id, parent, sha1, 1 FROM node WHERE id = ? + UNION ALL + SELECT n.id, n.parent, n.sha1, s.level + 1 + FROM node AS n JOIN subtree AS s ON n.parent = s.id + ) DELETE FROM node WHERE id IN ( + SELECT id FROM subtree WHERE level <> 1 AND sha1 IS NULL + AND id NOT IN (SELECT parent FROM node WHERE parent IS NOT NULL) + )`) + if err != nil { + return err + } + + for { + if result, err := stmt.Exec(dbParent); err != nil { + return err + } else if n, err := result.RowsAffected(); err != nil { + return err + } else if n == 0 { + return nil + } + } +} + +type syncPath struct { + db []string // database path, in terms of nodes + fs string // normalized filesystem path +} + +// syncResolveRoots normalizes filesystem paths given in command line arguments, +// and figures out a database path for each. Duplicates are skipped or rejected. +func syncResolveRoots(args []string, fullpaths bool) ( + roots []*syncPath, err error) { + for i := range args { + fs, err := filepath.Abs(filepath.Clean(args[i])) + if err != nil { + return nil, err + } + + roots = append(roots, + &syncPath{decodeWebPath(filepath.ToSlash(fs)), fs}) + } + + if fullpaths { + // Filter out duplicates. In this case, they're just duplicated work. + slices.SortFunc(roots, func(a, b *syncPath) int { + return strings.Compare(a.fs, b.fs) + }) + roots = slices.CompactFunc(roots, func(a, b *syncPath) bool { + if a.fs != b.fs && !strings.HasPrefix(b.fs, a.fs+"/") { + return false + } + log.Printf("asking to sync path twice: %s\n", b.fs) + return true + }) + } else { + // Keep just the basenames. + for _, path := range roots { + if len(path.db) > 0 { + path.db = path.db[len(path.db)-1:] + } + } + + // Different filesystem paths mapping to the same DB location + // are definitely a problem we would like to avoid, + // otherwise we don't care. + slices.SortFunc(roots, func(a, b *syncPath) int { + return slices.Compare(a.db, b.db) + }) + for i := 1; i < len(roots); i++ { + if slices.Equal(roots[i-1].db, roots[i].db) { + return nil, fmt.Errorf("duplicate root: %v", roots[i].db) + } + } + } + return +} + +const disposeCTE = `WITH RECURSIVE + root(id, sha1, parent, path) AS ( + SELECT id, sha1, parent, name FROM node WHERE id = ? + UNION ALL + SELECT r.id, r.sha1, n.parent, n.name || '/' || r.path + FROM node AS n JOIN root AS r ON n.id = r.parent + ), + children(id, sha1, path, level) AS ( + SELECT id, sha1, path, 1 FROM root WHERE parent IS NULL + UNION ALL + SELECT n.id, n.sha1, c.path || '/' || n.name, c.level + 1 + FROM node AS n JOIN children AS c ON n.parent = c.id + ), + removed(sha1, count, path) AS ( + SELECT sha1, COUNT(*) AS count, MIN(path) AS path + FROM children + GROUP BY sha1 + ), + orphaned(sha1, path, count, total) AS ( + SELECT r.sha1, r.path, r.count, COUNT(*) AS total + FROM removed AS r + JOIN node ON node.sha1 = r.sha1 + GROUP BY node.sha1 + HAVING count = total + )` + +// cmdSync ensures the given (sub)roots are accurately reflected +// in the database. +func cmdSync(fs *flag.FlagSet, args []string) error { + fullpaths := fs.Bool("fullpaths", false, "don't basename arguments") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 2 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + roots, err := syncResolveRoots(fs.Args()[1:], *fullpaths) + if err != nil { + return err + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + // In case of a failure during processing, the only retained side effects + // on the filesystem tree are: + // - Fixing dead symlinks to images. + // - Creating symlinks to images that aren't used by anything. + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + // Mild hack: upgrade the transaction to a write one straight away, + // in order to rule out deadlocks (preventable failure). + if _, err := tx.Exec(`END TRANSACTION; + BEGIN IMMEDIATE TRANSACTION`); err != nil { + return err + } + + c := syncContext{ctx: ctx, tx: tx, pb: newProgressBar(-1), + linked: make(map[string]struct{})} + defer c.pb.Stop() + + if c.stmtOrphan, err = c.tx.Prepare(disposeCTE + ` + INSERT OR IGNORE INTO orphan(sha1, path) + SELECT sha1, path FROM orphaned`); err != nil { + return err + } + if c.stmtDisposeSub, err = c.tx.Prepare(disposeCTE + ` + DELETE FROM node WHERE id + IN (SELECT DISTINCT id FROM children WHERE level <> 1)`); err != nil { + return err + } + if c.stmtDisposeAll, err = c.tx.Prepare(disposeCTE + ` + DELETE FROM node WHERE id + IN (SELECT DISTINCT id FROM children)`); err != nil { + return err + } + + // Info tasks take a position in the task semaphore channel. + // then fill the info channel. + // + // Immediately after syncDequeue(), the info channel is empty, + // but the semaphore might be full. + // + // By having at least one position in the info channel, + // we allow at least one info task to run to semaphore release, + // so that syncEnqueue() doesn't deadlock. + // + // By making it the same size as the semaphore, + // the end of this function doesn't need to dequeue while waiting. + // It also prevents goroutine leaks despite leaving them running-- + // once they finish their job, they're gone, + // and eventually the info channel would get garbage collected. + // + // The additional slot is there to handle the one result + // that may be placed while syncEnqueue() waits for the semaphore, + // i.e., it is for the result of the task that syncEnqueue() spawns. + c.info = make(chan syncFileInfo, cap(taskSemaphore)+1) + + for _, root := range roots { + if err := syncRoot(&c, root.db, root.fs); err != nil { + return err + } + } + return tx.Commit() +} + +// --- Removal ----------------------------------------------------------------- + +// cmdRemove is for manual removal of subtrees from the database. +// Beware that inputs are database, not filesystem paths. +func cmdRemove(fs *flag.FlagSet, args []string) error { + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 2 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + return err + } + defer tx.Rollback() + + for _, path := range fs.Args()[1:] { + var id sql.NullInt64 + for _, name := range decodeWebPath(path) { + if err := tx.QueryRow(`SELECT id FROM node + WHERE parent IS ? AND name = ?`, + id, name).Scan(&id); err != nil { + return err + } + } + if id.Int64 == 0 { + return errors.New("can't remove root") + } + + if _, err = tx.Exec(disposeCTE+` + INSERT OR IGNORE INTO orphan(sha1, path) + SELECT sha1, path FROM orphaned`, id); err != nil { + return err + } + if _, err = tx.Exec(disposeCTE+` + DELETE FROM node WHERE id + IN (SELECT DISTINCT id FROM children)`, id); err != nil { + return err + } + } + return tx.Commit() +} + +// --- Tagging ----------------------------------------------------------------- + +// cmdTag mass imports tags from data passed on stdin as a TSV +// of SHA1 TAG WEIGHT entries. +func cmdTag(fs *flag.FlagSet, args []string) error { + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 2 || fs.NArg() > 3 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + space := fs.Arg(1) + + var description sql.NullString + if fs.NArg() >= 3 { + description = sql.NullString{String: fs.Arg(2), Valid: true} + } + + // Note that starting as a write transaction prevents deadlocks. + // Imports are rare, and just bulk load data, so this scope is fine. + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if _, err := tx.Exec(`INSERT OR IGNORE INTO tag_space(name, description) + VALUES (?, ?)`, space, description); err != nil { + return err + } + + var spaceID int64 + if err := tx.QueryRow(`SELECT id FROM tag_space WHERE name = ?`, + space).Scan(&spaceID); err != nil { + return err + } + + // XXX: It might make sense to pre-erase all tag assignments within + // the given space for that image, the first time we see it: + // + // DELETE FROM tag_assignment + // WHERE sha1 = ? AND tag IN (SELECT id FROM tag WHERE space = ?) + // + // or even just clear the tag space completely: + // + // DELETE FROM tag_assignment + // WHERE tag IN (SELECT id FROM tag WHERE space = ?); + // DELETE FROM tag WHERE space = ?; + stmt, err := tx.Prepare(`INSERT INTO tag_assignment(sha1, tag, weight) + VALUES (?, (SELECT id FROM tag WHERE space = ? AND name = ?), ?) + ON CONFLICT DO UPDATE SET weight = ?`) + if err != nil { + return err + } + + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fields := strings.Split(scanner.Text(), "\t") + if len(fields) != 3 { + return errors.New("invalid input format") + } + + sha1, tag := fields[0], fields[1] + weight, err := strconv.ParseFloat(fields[2], 64) + if err != nil { + return err + } + + if _, err := tx.Exec( + `INSERT OR IGNORE INTO tag(space, name) VALUES (?, ?);`, + spaceID, tag); err != nil { + return nil + } + if _, err := stmt.Exec(sha1, spaceID, tag, weight, weight); err != nil { + log.Printf("%s: %s\n", sha1, err) + } + } + if err := scanner.Err(); err != nil { + return err + } + return tx.Commit() +} + +// --- Check ------------------------------------------------------------------- + +func isValidSHA1(hash string) bool { + if len(hash) != sha1.Size*2 || strings.ToLower(hash) != hash { + return false + } + if _, err := hex.DecodeString(hash); err != nil { + return false + } + return true +} + +func hashesToFileListing(root, suffix string, hashes []string) []string { + // Note that we're semi-duplicating {image,thumb}Path(). + paths := []string{root} + for _, hash := range hashes { + dir := filepath.Join(root, hash[:2]) + paths = append(paths, dir, filepath.Join(dir, hash+suffix)) + } + slices.Sort(paths) + return slices.Compact(paths) +} + +func collectFileListing(root string) (paths []string, err error) { + err = filepath.WalkDir(root, + func(path string, d fs.DirEntry, err error) error { + paths = append(paths, path) + return err + }) + + // Even though it should already be sorted somehow. + slices.Sort(paths) + return +} + +func checkFiles(root, suffix string, hashes []string) (bool, []string, error) { + db := hashesToFileListing(root, suffix, hashes) + fs, err := collectFileListing(root) + if err != nil { + return false, nil, err + } + + iDB, iFS, ok, intersection := 0, 0, true, []string{} + for iDB < len(db) && iFS < len(fs) { + if db[iDB] == fs[iFS] { + intersection = append(intersection, db[iDB]) + iDB++ + iFS++ + } else if db[iDB] < fs[iFS] { + ok = false + fmt.Printf("only in DB: %s\n", db[iDB]) + iDB++ + } else { + ok = false + fmt.Printf("only in FS: %s\n", fs[iFS]) + iFS++ + } + } + for _, path := range db[iDB:] { + ok = false + fmt.Printf("only in DB: %s\n", path) + } + for _, path := range fs[iFS:] { + ok = false + fmt.Printf("only in FS: %s\n", path) + } + return ok, intersection, nil +} + +func checkHash(path string) (message string, err error) { + f, err := os.Open(path) + if err != nil { + return err.Error(), nil + } + defer f.Close() + + // We get 2 levels of parent directories in here, just filter them out. + if fi, err := f.Stat(); err != nil { + return err.Error(), nil + } else if fi.IsDir() { + return "", nil + } + + hash := sha1.New() + _, err = io.CopyBuffer(hash, f, make([]byte, 65536)) + if err != nil { + return err.Error(), nil + } + + sha1 := hex.EncodeToString(hash.Sum(nil)) + if sha1 != filepath.Base(path) { + return fmt.Sprintf("mismatch, found %s", sha1), nil + } + return "", nil +} + +func checkHashes(paths []string) (bool, error) { + log.Println("checking image hashes") + var failed atomic.Bool + err := parallelize(paths, func(path string) (string, error) { + message, err := checkHash(path) + if message != "" { + failed.Store(true) + } + return message, err + }) + return !failed.Load(), err +} + +// cmdCheck carries out various database consistency checks. +func cmdCheck(fs *flag.FlagSet, args []string) error { + full := fs.Bool("full", false, "verify image hashes") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + // Check if hashes are in the right format. + log.Println("checking image hashes") + + allSHA1, err := dbCollectStrings(`SELECT sha1 FROM image`) + if err != nil { + return err + } + + ok := true + for _, hash := range allSHA1 { + if !isValidSHA1(hash) { + ok = false + fmt.Printf("invalid image SHA1: %s\n", hash) + } + } + + // This is, rather obviously, just a strict subset. + // Although it doesn't run in the same transaction. + thumbSHA1, err := dbCollectStrings(`SELECT sha1 FROM image + WHERE thumbw IS NOT NULL OR thumbh IS NOT NULL`) + if err != nil { + return err + } + + // This somewhat duplicates {image,thumb}Path(). + log.Println("checking SQL against filesystem") + okImages, intersection, err := checkFiles( + filepath.Join(galleryDirectory, nameOfImageRoot), "", allSHA1) + if err != nil { + return err + } + + okThumbs, _, err := checkFiles( + filepath.Join(galleryDirectory, nameOfThumbRoot), ".webp", thumbSHA1) + if err != nil { + return err + } + if !okImages || !okThumbs { + ok = false + } + + log.Println("checking for dead symlinks") + for _, path := range intersection { + if _, err := os.Stat(path); err != nil { + ok = false + fmt.Printf("%s: %s\n", path, err) + } + } + + if *full { + if ok2, err := checkHashes(intersection); err != nil { + return err + } else if !ok2 { + ok = false + } + } + + if !ok { + return errors.New("detected inconsistencies") + } + return nil +} + +// --- Thumbnailing ------------------------------------------------------------ + +func identifyThumbnail(path string) (w, h int, err error) { + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + + config, err := webp.DecodeConfig(f) + if err != nil { + return + } + return config.Width, config.Height, nil +} + +func makeThumbnail(load bool, pathImage, pathThumb string) ( + w, h int, err error) { + if load { + if w, h, err = identifyThumbnail(pathThumb); err == nil { + return + } + } + + thumbDirname, _ := filepath.Split(pathThumb) + if err := os.MkdirAll(thumbDirname, 0755); err != nil { + return 0, 0, err + } + + // Create a normalized thumbnail. Since we don't particularly need + // any complex processing, such as surrounding of metadata, + // simply push it through ImageMagick. + // + // - http://www.ericbrasseur.org/gamma.html + // - https://www.imagemagick.org/Usage/thumbnails/ + // - https://imagemagick.org/script/command-line-options.php#layers + // + // "info:" output is written for each frame, which is why we delete + // all of them but the first one beforehands. + // + // TODO: See if we can optimize resulting WebP animations. + // (Do -layers optimize* apply to this format at all?) + cmd := exec.Command("magick", "-limit", "thread", "1", pathImage, + "-coalesce", "-colorspace", "RGB", "-auto-orient", "-strip", + "-resize", "256x128>", "-colorspace", "sRGB", + "-format", "%w %h", "+write", pathThumb, "-delete", "1--1", "info:") + + out, err := cmd.Output() + if err != nil { + return 0, 0, err + } + + _, err = fmt.Fscanf(bytes.NewReader(out), "%d %d", &w, &h) + return w, h, err +} + +// cmdThumbnail generates missing thumbnails, in parallel. +func cmdThumbnail(fs *flag.FlagSet, args []string) error { + load := fs.Bool("load", false, "try to load existing thumbnail files") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + hexSHA1 := fs.Args()[1:] + if len(hexSHA1) == 0 { + // Get all unique images in the database with no thumbnail. + var err error + hexSHA1, err = dbCollectStrings(`SELECT sha1 FROM image + WHERE thumbw IS NULL OR thumbh IS NULL`) + if err != nil { + return err + } + } + + stmt, err := db.Prepare( + `UPDATE image SET thumbw = ?, thumbh = ? WHERE sha1 = ?`) + if err != nil { + return err + } + defer stmt.Close() + + var mu sync.Mutex + return parallelize(hexSHA1, func(sha1 string) (message string, err error) { + pathImage := imagePath(sha1) + pathThumb := thumbPath(sha1) + w, h, err := makeThumbnail(*load, pathImage, pathThumb) + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return string(ee.Stderr), nil + } + return "", err + } + + mu.Lock() + defer mu.Unlock() + _, err = stmt.Exec(w, h, sha1) + return "", err + }) +} + +// --- Perceptual hash --------------------------------------------------------- + +type linearImage struct { + img image.Image +} + +func newLinearImage(img image.Image) *linearImage { + return &linearImage{img: img} +} + +func (l *linearImage) ColorModel() color.Model { return l.img.ColorModel() } +func (l *linearImage) Bounds() image.Rectangle { return l.img.Bounds() } + +func unSRGB(c uint32) uint8 { + n := float64(c) / 0xffff + if n <= 0.04045 { + return uint8(n * (255.0 / 12.92)) + } + return uint8(math.Pow((n+0.055)/(1.055), 2.4) * 255.0) +} + +func (l *linearImage) At(x, y int) color.Color { + r, g, b, a := l.img.At(x, y).RGBA() + return color.RGBA{ + R: unSRGB(r), G: unSRGB(g), B: unSRGB(b), A: uint8(a >> 8)} +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +// isWebPAnimation returns whether the given ReadSeeker starts a WebP animation. +// See https://developers.google.com/speed/webp/docs/riff_container +func isWebPAnimation(rs io.ReadSeeker) (bool, error) { + b := make([]byte, 21) + if _, err := rs.Read(b); err != nil { + return false, err + } + if _, err := rs.Seek(0, io.SeekStart); err != nil { + return false, err + } + + return bytes.Equal(b[:4], []byte("RIFF")) && + bytes.Equal(b[8:16], []byte("WEBPVP8X")) && + b[20]&0b00000010 != 0, nil +} + +var errIsAnimation = errors.New("cannot perceptually hash animations") + +func dhashWebP(rs io.ReadSeeker) (uint64, error) { + if a, err := isWebPAnimation(rs); err != nil { + return 0, err + } else if a { + return 0, errIsAnimation + } + + // Doing this entire thing in Go is SLOW, but convenient. + source, err := webp.Decode(rs) + if err != nil { + return 0, err + } + + var ( + linear = newLinearImage(source) + resized = image.NewNRGBA64(image.Rect(0, 0, 9, 8)) + ) + draw.CatmullRom.Scale(resized, resized.Bounds(), + linear, linear.Bounds(), draw.Src, nil) + + var hash uint64 + for y := 0; y < 8; y++ { + var grey [9]float32 + for x := 0; x < 9; x++ { + rgba := resized.NRGBA64At(x, y) + grey[x] = 0.2126*float32(rgba.R) + + 0.7152*float32(rgba.G) + + 0.0722*float32(rgba.B) + } + + var row uint64 + if grey[0] < grey[1] { + row |= 1 << 7 + } + if grey[1] < grey[2] { + row |= 1 << 6 + } + if grey[2] < grey[3] { + row |= 1 << 5 + } + if grey[3] < grey[4] { + row |= 1 << 4 + } + if grey[4] < grey[5] { + row |= 1 << 3 + } + if grey[5] < grey[6] { + row |= 1 << 2 + } + if grey[6] < grey[7] { + row |= 1 << 1 + } + if grey[7] < grey[8] { + row |= 1 << 0 + } + hash = hash<<8 | row + } + return hash, nil +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +func makeDhash(sha1 string) (uint64, error) { + pathThumb := thumbPath(sha1) + f, err := os.Open(pathThumb) + if err != nil { + return 0, err + } + defer f.Close() + return dhashWebP(f) +} + +// cmdDhash computes perceptual hashes from thumbnails. +func cmdDhash(fs *flag.FlagSet, args []string) error { + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 { + return errWrongUsage + } + if err := openDB(fs.Arg(0)); err != nil { + return err + } + + hexSHA1 := fs.Args()[1:] + if len(hexSHA1) == 0 { + var err error + hexSHA1, err = dbCollectStrings(`SELECT sha1 FROM image + WHERE thumbw IS NOT NULL AND thumbh IS NOT NULL AND dhash IS NULL`) + if err != nil { + return err + } + } + + stmt, err := db.Prepare(`UPDATE image SET dhash = ? WHERE sha1 = ?`) + if err != nil { + return err + } + defer stmt.Close() + + var mu sync.Mutex + return parallelize(hexSHA1, func(sha1 string) (message string, err error) { + hash, err := makeDhash(sha1) + if errors.Is(err, errIsAnimation) { + // Ignoring this common condition. + return "", nil + } else if err != nil { + return err.Error(), nil + } + + mu.Lock() + defer mu.Unlock() + _, err = stmt.Exec(int64(hash), sha1) + return "", err + }) +} + +// --- Main -------------------------------------------------------------------- + +var errWrongUsage = errors.New("wrong usage") + +var commands = map[string]struct { + handler func(*flag.FlagSet, []string) error + usage string + function string +}{ + "init": {cmdInit, "GD", "Initialize a database."}, + "web": {cmdWeb, "GD ADDRESS", "Launch a web interface."}, + "tag": {cmdTag, "GD SPACE [DESCRIPTION]", "Import tags."}, + "sync": {cmdSync, "GD ROOT...", "Synchronise with the filesystem."}, + "remove": {cmdRemove, "GD PATH...", "Remove database subtrees."}, + "check": {cmdCheck, "GD", "Run consistency checks."}, + "thumbnail": {cmdThumbnail, "GD [SHA1...]", "Generate thumbnails."}, + "dhash": {cmdDhash, "GD [SHA1...]", "Compute perceptual hashes."}, +} + +func usage() { + f := flag.CommandLine.Output() + fmt.Fprintf(f, "Usage: %s COMMAND [ARG...]\n", os.Args[0]) + flag.PrintDefaults() + + // The alphabetic ordering is unfortunate, but tolerable. + keys := []string{} + for key := range commands { + keys = append(keys, key) + } + sort.Strings(keys) + + fmt.Fprintf(f, "\nCommands:\n") + for _, key := range keys { + fmt.Fprintf(f, " %s [OPTION...] %s\n \t%s\n", + key, commands[key].usage, commands[key].function) + } +} + +func main() { + // This implements the -h switch for us by default. + // The rest of the handling here closely follows what flag does internally. + flag.Usage = usage + flag.Parse() + if flag.NArg() < 1 { + flag.Usage() + os.Exit(2) + } + + cmd, ok := commands[flag.Arg(0)] + if !ok { + fmt.Fprintf(flag.CommandLine.Output(), + "unknown command: %s\n", flag.Arg(0)) + flag.Usage() + os.Exit(2) + } + + fs := flag.NewFlagSet(flag.Arg(0), flag.ExitOnError) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), + "Usage: %s [OPTION...] %s\n%s\n", + fs.Name(), cmd.usage, cmd.function) + fs.PrintDefaults() + } + + taskSemaphore = newSemaphore(runtime.NumCPU()) + err := cmd.handler(fs, flag.Args()[1:]) + + // Note that the database object has a closing finalizer, + // we just additionally print any errors coming from there. + if db != nil { + if err := db.Close(); err != nil { + log.Println(err) + } + } + + if errors.Is(err, errWrongUsage) { + fs.Usage() + os.Exit(2) + } else if err != nil { + log.Fatalln(err) + } +} diff --git a/public/gallery.js b/public/gallery.js new file mode 100644 index 0000000..9d3b067 --- /dev/null +++ b/public/gallery.js @@ -0,0 +1,675 @@ +'use strict' + +let callActive = false +let callFaulty = false + +function call(method, params) { + // XXX: At least with POST, unsuccessful requests result + // in catched errors containing Errors with a null message. + // This is an issue within XMLHttpRequest. + callActive++ + return m.request({ + method: "POST", + url: `/api/${method}`, + body: params, + }).then(result => { + callActive-- + callFaulty = false + return result + }).catch(error => { + callActive-- + callFaulty = true + throw error + }) +} + +const loading = (window.location.hostname !== 'localhost') ? 'lazy' : undefined + +let Header = { + global: [ + {name: "Browse", route: '/browse'}, + {name: "Tags", route: '/tags'}, + {name: "Duplicates", route: '/duplicates'}, + {name: "Orphans", route: '/orphans'}, + ], + + image: [ + { + route: '/view', + render: () => m(m.route.Link, { + href: `/view/:key`, + params: {key: m.route.param('key')}, + class: m.route.get().startsWith('/view') + ? 'active' : undefined, + }, "View"), + }, + { + route: '/similar', + render: () => m(m.route.Link, { + href: `/similar/:key`, + params: {key: m.route.param('key')}, + class: m.route.get().startsWith('/similar') + ? 'active' : undefined, + }, "Similar"), + }, + ], + + search: [ + { + route: '/search', + render: () => m(m.route.Link, { + href: `/search/:key`, + params: {key: m.route.param('key')}, + class: m.route.get().startsWith('/search') + ? 'active' : undefined, + }, "Search"), + }, + ], + + view(vnode) { + const route = m.route.get() + const main = this.global.map(x => + m(m.route.Link, { + href: x.route, + class: route.startsWith(x.route) ? 'active' : undefined, + }, x.name)) + + let context + if (this.image.some(x => route.startsWith(x.route))) + context = this.image.map(x => x.render()) + if (this.search.some(x => route.startsWith(x.route))) + context = this.search.map(x => x.render()) + + return m('.header', {}, [ + m('nav', main), + m('nav', context), + callFaulty + ? m('.activity.error[title=Error]', '●') + : callActive + ? m('.activity[title=Busy]', '●') + : m('.activity[title=Idle]', '○'), + ]) + }, +} + +let Thumbnail = { + view(vnode) { + const e = vnode.attrs.info + if (!e.thumbW || !e.thumbH) + return m('.thumbnail.missing', {...vnode.attrs, info: null}) + return m('img.thumbnail', {...vnode.attrs, info: null, + src: `/thumb/${e.sha1}`, width: e.thumbW, height: e.thumbH, + loading}) + }, +} + +let ScoredTag = { + view(vnode) { + const {space, tagname, score} = vnode.attrs + return m('li', [ + m("meter[max=1.0]", {value: score, title: score}, score), + ` `, + m(m.route.Link, { + href: `/search/:key`, + params: {key: `${space}:${tagname}`}, + }, ` ${tagname}`), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let BrowseModel = { + path: undefined, + subdirectories: [], + entries: [], + collator: new Intl.Collator(undefined, {numeric: true}), + + async reload(path) { + if (this.path !== path) { + this.path = path + this.subdirectories = [] + this.entries = [] + } + + let resp = await call('browse', {path}) + this.subdirectories = resp.subdirectories + this.entries = resp.entries.sort((a, b) => + this.collator.compare(a.name, b.name)) + }, + + joinPath(parent, child) { + if (!parent) + return child + if (!child) + return parent + return `${parent}/${child}` + }, + + getBrowseLinks() { + if (this.path === undefined) + return [] + + let links = [{name: "Root", path: "", level: -1}], path + for (const crumb of this.path.split('/').filter(s => !!s)) { + path = this.joinPath(path, crumb) + links.push({name: crumb, path: path, level: -1}) + } + + links[links.length - 1].level = 0 + + for (const sub of this.subdirectories) { + links.push( + {name: sub, path: this.joinPath(this.path, sub), level: +1}) + } + return links + }, +} + +let BrowseBarLink = { + view(vnode) { + const link = vnode.attrs.link + + let c = 'selected' + if (link.level < 0) + c = 'parent' + if (link.level > 0) + c = 'child' + + return m('li', { + class: c, + }, m(m.route.Link, { + href: `/browse/:key`, + params: {key: link.path}, + }, link.name)) + }, +} + +let BrowseView = { + // So that Page Up/Down, etc., work after changing directories. + // Programmatically focusing a scrollable element requires setting tabindex, + // and causes :focus-visible on page load, which we suppress in CSS. + // I wish there was another way, but the workaround isn't particularly bad. + // focus({focusVisible: true}) is FF 104+ only and experimental. + oncreate(vnode) { vnode.dom.focus() }, + + view(vnode) { + return m('.browser[tabindex=0]', { + // Trying to force the oncreate on path changes. + key: BrowseModel.path, + }, BrowseModel.entries.map(info => { + return m(m.route.Link, {href: `/view/${info.sha1}`}, + m(Thumbnail, {info, title: info.name})) + })) + }, +} + +let Browse = { + // Reload the model immediately, to improve responsivity. + // But we don't need to: https://mithril.js.org/route.html#preloading-data + // Also see: https://mithril.js.org/route.html#route-cancellation--blocking + oninit(vnode) { + let path = vnode.attrs.key || "" + BrowseModel.reload(path) + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, [ + m('.sidebar', [ + m('ul.path', BrowseModel.getBrowseLinks() + .map(link => m(BrowseBarLink, {link}))), + ]), + m(BrowseView), + ]), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let TagsModel = { + ns: null, + namespaces: {}, + + async reload(ns) { + if (this.ns !== ns) { + this.ns = ns + this.namespaces = {} + } + + this.namespaces = await call('tags', {namespace: ns}) + }, +} + +let TagsList = { + view(vnode) { + // TODO: Make it possible to sort by count. + const tags = Object.entries(vnode.attrs.tags) + .sort(([a, b]) => a[0].localeCompare(b[0])) + + return (tags.length == 0) + ? "No tags" + : m("ul", tags.map(([name, count]) => m("li", [ + m(m.route.Link, { + href: `/search/:key`, + params: {key: `${vnode.attrs.space}:${name}`}, + }, ` ${name}`), + ` ×${count}`, + ]))) + }, +} + +let TagsView = { + // See BrowseView. + oncreate(vnode) { vnode.dom.focus() }, + + view(vnode) { + // XXX: The empty-named tag namespace gets a bit shafted, + // in particular in the router, as well as with its header. + // Maybe we could refer to it by its numeric ID in routing. + const names = Object.keys(TagsModel.namespaces) + .sort((a, b) => a.localeCompare(b)) + + let children = (names.length == 0) + ? "No namespaces" + : names.map(space => { + const ns = TagsModel.namespaces[space] + return [ + m("h2", space), + ns.description ? m("p", ns.description) : [], + m(TagsList, {space, tags: ns.tags}), + ] + }) + return m('.tags[tabindex=0]', {}, children) + }, +} + +let Tags = { + oninit(vnode) { + let ns = vnode.attrs.key + TagsModel.reload(ns) + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, m(TagsView)), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let DuplicatesModel = { + entries: [], + + async reload() { + this.entries = await call('duplicates', {}) + }, +} + +let DuplicatesThumbnail = { + view(vnode) { + const info = vnode.attrs.info + return [ + m(m.route.Link, {href: `/similar/${info.sha1}`}, + m(Thumbnail, {info})), + (info.occurences != 1) ? ` ×${info.occurences}` : [], + ] + }, +} + +let DuplicatesList = { + // See BrowseView. + oncreate(vnode) { vnode.dom.focus() }, + + view(vnode) { + let children = (DuplicatesModel.entries.length == 0) + ? "No duplicates" + : DuplicatesModel.entries.map(group => + m('.row', group.map(entry => + m(DuplicatesThumbnail, {info: entry})))) + return m('.duplicates[tabindex=0]', {}, children) + }, +} + +let Duplicates = { + oninit(vnode) { + DuplicatesModel.reload() + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, m(DuplicatesList)), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let OrphansModel = { + entries: [], + + async reload() { + this.entries = await call('orphans', {}) + }, +} + +let OrphansReplacement = { + view(vnode) { + const info = vnode.attrs.info + if (!info) + return [] + + return [ + ` → `, + m(m.route.Link, {href: `/view/${info.sha1}`}, + m(Thumbnail, {info})), + `${info.tags} tags`, + ] + }, +} + +let OrphansRow = { + view(vnode) { + const info = vnode.attrs.info + return m('.row', [ + // It might not load, but still allow tag viewing. + m(m.route.Link, {href: `/view/${info.sha1}`}, + m(Thumbnail, {info})), + `${info.tags} tags`, + m(OrphansReplacement, {info: info.replacement}), + ]) + }, +} + +let OrphansList = { + // See BrowseView. + oncreate(vnode) { vnode.dom.focus() }, + + view(vnode) { + let children = (OrphansModel.entries.length == 0) + ? "No orphans" + : OrphansModel.entries.map(info => [ + m("h2", info.lastPath), + m(OrphansRow, {info}), + ]) + return m('.orphans[tabindex=0]', {}, children) + }, +} + +let Orphans = { + oninit(vnode) { + OrphansModel.reload() + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, m(OrphansList)), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let ViewModel = { + sha1: undefined, + width: 0, + height: 0, + paths: [], + tags: {}, + + async reload(sha1) { + if (this.sha1 !== sha1) { + this.sha1 = sha1 + this.width = this.height = 0 + this.paths = [] + this.tags = {} + } + + let resp = await call('info', {sha1: sha1}) + this.width = resp.width + this.height = resp.height + this.paths = resp.paths + this.tags = resp.tags + }, +} + +let ViewBarBrowseLink = { + view(vnode) { + return m(m.route.Link, { + href: `/browse/:key`, + params: {key: vnode.attrs.path}, + }, vnode.attrs.name) + }, +} + +let ViewBarPath = { + view(vnode) { + const parents = vnode.attrs.path.split('/') + const basename = parents.pop() + + let result = [], path + if (!parents.length) + result.push(m(ViewBarBrowseLink, {path: "", name: "Root"}), "/") + for (const crumb of parents) { + path = BrowseModel.joinPath(path, crumb) + result.push(m(ViewBarBrowseLink, {path, name: crumb}), "/") + } + result.push(basename) + return result + }, +} + +let ViewBar = { + view(vnode) { + return m('.viewbar', [ + m('h2', "Locations"), + m('ul', ViewModel.paths.map(path => + m('li', m(ViewBarPath, {path})))), + m('h2', "Tags"), + Object.entries(ViewModel.tags).map(([space, tags]) => [ + m("h3", m(m.route.Link, {href: `/tags/${space}`}, space)), + m("ul.tags", Object.entries(tags) + .sort(([t1, w1], [t2, w2]) => (w2 - w1)) + .map(([tag, score]) => + m(ScoredTag, {space, tagname: tag, score}))), + ]), + ]) + }, +} + +let View = { + oninit(vnode) { + let sha1 = vnode.attrs.key || "" + ViewModel.reload(sha1) + }, + + view(vnode) { + const view = m('.view', [ + ViewModel.sha1 !== undefined + ? m('img', {src: `/image/${ViewModel.sha1}`, + width: ViewModel.width, height: ViewModel.height}) + : "No image.", + ]) + return m('.container', {}, [ + m(Header), + m('.body', {}, [view, m(ViewBar)]), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let SimilarModel = { + sha1: undefined, + info: {paths: []}, + groups: {}, + + async reload(sha1) { + if (this.sha1 !== sha1) { + this.sha1 = sha1 + this.info = {paths: []} + this.groups = {} + } + + let resp = await call('similar', {sha1: sha1}) + this.info = resp.info + this.groups = resp.groups + }, +} + +let SimilarThumbnail = { + view(vnode) { + const info = vnode.attrs.info + return m(m.route.Link, {href: `/view/${info.sha1}`}, + m(Thumbnail, {info})) + }, +} + +let SimilarGroup = { + view(vnode) { + const images = vnode.attrs.images + let result = [ + m('h2', vnode.attrs.name), + images.map(info => m('.row', [ + m(SimilarThumbnail, {info}), + m('ul', [ + m('li', Math.round(info.pixelsRatio * 100) + + "% pixels of input image"), + info.paths.map(path => + m('li', m(ViewBarPath, {path}))), + ]), + ])) + ] + if (!images.length) + result.push("No matches.") + return result + }, +} + +let SimilarList = { + view(vnode) { + if (SimilarModel.sha1 === undefined || + SimilarModel.info.paths.length == 0) + return "No image" + + const info = SimilarModel.info + return m('.similar', {}, [ + m('.row', [ + m(SimilarThumbnail, {info}), + m('ul', info.paths.map(path => + m('li', m(ViewBarPath, {path})))), + ]), + Object.entries(SimilarModel.groups).map(([name, images]) => + m(SimilarGroup, {name, images})), + ]) + }, +} + +let Similar = { + oninit(vnode) { + let sha1 = vnode.attrs.key || "" + SimilarModel.reload(sha1) + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, m(SimilarList)), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +let SearchModel = { + query: undefined, + matches: [], + related: {}, + + async reload(query) { + if (this.query !== query) { + this.query = query + this.matches = [] + this.related = {} + } + + let resp = await call('search', {query}) + this.matches = resp.matches + this.related = resp.related + }, +} + +let SearchRelated = { + view(vnode) { + return Object.entries(SearchModel.related) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([space, tags]) => [ + m('h2', space), + m('ul.tags', tags + .sort((a, b) => (b.score - a.score)) + .map(({tag, score}) => + m(ScoredTag, {space, tagname: tag, score}))), + ]) + }, +} + +let SearchView = { + // See BrowseView. + oncreate(vnode) { vnode.dom.focus() }, + + view(vnode) { + return m('.browser[tabindex=0]', { + // Trying to force the oncreate on path changes. + key: SearchModel.path, + }, SearchModel.matches + .sort((a, b) => b.score - a.score) + .map(info => { + return m(m.route.Link, {href: `/view/${info.sha1}`}, + m(Thumbnail, {info, title: info.score})) + })) + }, +} + +let Search = { + oninit(vnode) { + SearchModel.reload(vnode.attrs.key) + }, + + view(vnode) { + return m('.container', {}, [ + m(Header), + m('.body', {}, [ + m('.sidebar', [ + m('p', SearchModel.query), + m(SearchRelated), + ]), + m(SearchView), + ]), + ]) + }, +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +window.addEventListener('load', () => { + m.route(document.body, "/browse/", { + // The path doesn't need to be escaped, perhaps change that (":key..."). + "/browse/": Browse, + "/browse/:key": Browse, + "/tags": Tags, + "/tags/:key": Tags, + "/duplicates": Duplicates, + "/orphans": Orphans, + + "/view/:key": View, + "/similar/:key": Similar, + + "/search/:key": Search, + }) +}) diff --git a/public/style.css b/public/style.css new file mode 100644 index 0000000..1bdeb3f --- /dev/null +++ b/public/style.css @@ -0,0 +1,102 @@ +:root { --shade-color: #eee; } + +body { margin: 0; padding: 0; font-family: sans-serif; } +a { color: inherit; } + +.container { display: flex; flex-direction: column; + height: 100vh; width: 100vw; overflow: hidden; } + +.body { display: flex; flex-grow: 1; overflow: hidden; position: relative; } +.body::after { content: ''; position: absolute; pointer-events: none; + top: 0; left: 0; right: 0; height: .75rem; + background: linear-gradient(#fff, rgb(255 255 255 / 0%)); } + +.header { color: #000; background: #aaa linear-gradient(#888, #999); + display: flex; justify-content: space-between; column-gap: .5rem; } +.header nav { display: flex; margin: 0 .5rem; align-items: end; } +.header nav a { display: block; text-decoration: none; + background: #bbb linear-gradient(#bbb, #ccc); + margin: .25rem 0 0 -1px; padding: .25rem .75rem; + border: 1px solid #888; border-radius: .5rem .5rem 0 0; } +.header nav a.active { font-weight: bold; border-bottom: 1px solid #fff; + background: #fff linear-gradient(#eee, #fff); } +.header nav a.active, .header nav a:hover { padding-bottom: .4rem; } +.header .activity { padding: .25rem .5rem; align-self: center; color: #fff; } +.header .activity.error { color: #f00; } + +.sidebar { padding: .25rem .5rem; background: var(--shade-color); + border-right: 1px solid #ccc; overflow: auto; + min-width: 10rem; max-width: 20rem; flex-shrink: 0; } +.sidebar h2 { margin: 0.5em 0 0.25em 0; padding: 0; font-size: 1.2rem; } +.sidebar ul { margin: .5rem 0; padding: 0; } + +.sidebar .path { margin: .5rem -.5rem; } +.sidebar .path li { margin: 0; padding: 0; } +.sidebar .path li a { padding: .25rem .5rem; padding-left: 30px; + display: block; text-decoration: none; white-space: nowrap; } +.sidebar .path li a:hover { background-color: rgb(0 0 0 / 10%); } + +.sidebar .path li.parent a { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Cpath d='M 4 14 10 8 16 14' stroke='%23888' stroke-width='4' fill='none' /%3E%3C/svg%3E%0A"); + background-repeat: no-repeat; background-position: 5px center; } + +.sidebar .path li.selected a { font-weight: bold; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Ccircle cx='10' cy='10' r='6' fill='%23888' /%3E%3C/svg%3E%0A"); + background-repeat: no-repeat; background-position: 5px center; } + +.sidebar .path li.child a { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='20' height='20'%3E%3Cpath d='M 4 6 10 12 16 6' stroke='%23888' stroke-width='4' fill='none' /%3E%3C/svg%3E%0A"); + background-repeat: no-repeat; background-position: 5px center; } + +.browser { overflow: auto; display: flex; flex-wrap: wrap; + align-content: flex-start; justify-content: center; align-items: center; + gap: 3px; padding: 9px; flex-grow: 1; } +.browser:focus-visible { outline: 0; box-shadow: none; } + +.tags { padding: .5rem; flex-grow: 1; overflow: auto; } +.tags:focus-visible { outline: 0; box-shadow: none; } +.tags h2 { margin: .5em 0 .25em 0; padding: 0; font-size: 1.1rem; } +.tags p { margin: .25em 0; } +.tags ul { display: flex; margin: .5em 0; padding: 0; + flex-wrap: wrap; gap: .25em; } +.tags ul li { display: block; margin: 0; padding: .25em .5em; + border-radius: .5rem; background: var(--shade-color); } + +img.thumbnail { display: block; + background: repeating-conic-gradient(#eee 0% 25%, transparent 0% 50%) + 50% / 20px 20px; } +img.thumbnail, .thumbnail.missing { box-shadow: 0 0 3px rgba(0, 0, 0, 0.75); + margin: 3px; border: 0px solid #000; } +.thumbnail.missing { width: 128px; height: 128px; position: relative; } +.thumbnail.missing::after { content: '?'; font-size: 64px; + position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); } + +.view { display: flex; flex-grow: 1; overflow: hidden; + justify-content: center; align-items: center; } +.view img { max-width: 100%; max-height: 100%; object-fit: contain; } +.view img { z-index: 1; } + +.viewbar { padding: .25rem .5rem; background: #eee; + border-left: 1px solid #ccc; min-width: 20rem; overflow: auto; } +.viewbar h2 { margin: 0.5em 0 0.25em 0; padding: 0; font-size: 1.2rem; } +.viewbar h3 { margin: 0.25em 0; padding: 0; font-size: 1.1rem; } +.viewbar ul { margin: 0; padding: 0 0 0 1.25em; list-style-type: "- "; } +.viewbar ul.tags { padding: 0; list-style-type: none; } +.viewbar li { margin: 0; padding: 0; } + +.sidebar meter, +.viewbar meter { width: 1.25rem; + /* background: white; border: 1px solid #ccc; */ } + +.similar { padding: .5rem; flex-grow: 1; overflow: auto; } +.similar h2 { margin: 1em 0 0.5em 0; padding: 0; font-size: 1.2rem; } +.similar .row { display: flex; margin: .5rem 0; } +.similar .row ul { margin: 0; padding: 0 0 0 1.25em; list-style-type: "- "; } + +.duplicates, +.orphans { padding: .5rem; flex-grow: 1; overflow: auto; } +.duplicates .row, +.orphans .row { display: flex; margin: .5rem 0; align-items: center; gap: 3px; } + +.orphans .row { margin-bottom: 1.25rem; } +.orphans h2 { margin: 0.25em 0; padding: 0; font-size: 1.1rem; } diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..2f12d07 --- /dev/null +++ b/test.sh @@ -0,0 +1,65 @@ +#!/bin/sh -xe +cd "$(dirname "$0")" +make gallery +target=/tmp/G input=/tmp/G/Test +rm -rf $target + +mkdir -p $target $input/Test $input/Empty +gen() { magick "$@"; sha1=$(sha1sum "$(eval echo \$\{$#\})" | cut -d' ' -f1); } + +gen wizard: $input/wizard.webp +gen -seed 10 -size 256x256 plasma:fractal \ + $input/Test/dhash.jpg +gen -seed 10 -size 256x256 plasma:fractal \ + $input/Test/dhash.png +sha1duplicate=$sha1 +cp $input/Test/dhash.png \ + $input/Test/multiple-paths.png + +gen -seed 20 -size 160x128 plasma:fractal \ + -bordercolor transparent -border 64 \ + $input/Test/transparent-wide.png +gen -seed 30 -size 1024x256 plasma:fractal \ + -alpha set -channel A -evaluate multiply 0.2 \ + $input/Test/translucent-superwide.png + +gen -size 96x96 -delay 10 -loop 0 \ + -seed 111 plasma:fractal \ + -seed 222 plasma:fractal \ + -seed 333 plasma:fractal \ + -seed 444 plasma:fractal \ + -seed 555 plasma:fractal \ + -seed 666 plasma:fractal \ + $input/Test/animation-small.gif +sha1animated=$sha1 +gen $input/Test/animation-small.gif \ + $input/Test/video.mp4 + +./gallery init $target +./gallery sync $target $input "$@" +./gallery thumbnail $target +./gallery dhash $target +./gallery tag $target test "Test space" <<-END + $sha1duplicate foo 1.0 + $sha1duplicate bar 0.5 + $sha1animated foo 0.8 +END + +# TODO: Test all the various possible sync transitions. +mv $input/Test $input/Plasma +./gallery sync $target $input + +./gallery web $target :8080 & +web=$! +trap "kill $web; wait $web" EXIT INT TERM +sleep 0.25 + +call() (curl http://localhost:8080/api/$1 -X POST --data-binary @-) + +# TODO: Verify that things are how we expect them to be. +echo '{"path":"'"$(basename "$input")"'"}' | call browse +echo '{}' | call tags +echo '{}' | call duplicates +echo '{}' | call orphans +echo '{"sha1":"'"$sha1duplicate"'"}' | call info +echo '{"sha1":"'"$sha1duplicate"'"}' | call similar