gallery: try to avoid OOM while thumbnailing

iotop showed gigabytes of writes for a DB in the order of 100 MB.

deeptagger/deeptagger.cpp

@@ -309,8 +309,8 @@ run(std::vector<Magick::Image> &images, const Config &config,
 			if (config.sigmoid)
 				value = 1 / (1 + std::exp(-value));
 			if (value > g.threshold) {
-				printf("%s\t%.2f\t%s\n", images.at(i).fileName().c_str(),
-					value, config.tags.at(t).c_str());
+				printf("%s\t%s\t%.2f\n", images.at(i).fileName().c_str(),
+					config.tags.at(t).c_str(), value);
 			}
 		}
 	}

main.go

@@ -41,6 +41,9 @@ import (
 	"golang.org/x/image/webp"
 )
 
+// #include <unistd.h>
+import "C"
+
 var (
 	db               *sql.DB // sqlite database
 	galleryDirectory string  // gallery directory
@@ -1415,7 +1418,11 @@ func syncPostProcess(c *syncContext, info syncFileInfo) error {
 	case info.err != nil:
 		// * → error
 		if ee, ok := info.err.(*exec.ExitError); ok {
-			syncPrintf(c, "%s: %s", info.fsPath, ee.Stderr)
+			message := string(ee.Stderr)
+			if message == "" {
+				message = ee.String()
+			}
+			syncPrintf(c, "%s: %s", info.fsPath, message)
 		} else {
 			return info.err
 		}
@@ -2172,8 +2179,13 @@ func makeThumbnail(load bool, pathImage, pathThumb string) (
 		return 0, 0, err
 	}
 
+	// This is still too much, but it will be effective enough.
+	memoryLimit := strconv.FormatInt(
+		int64(C.sysconf(C._SC_PHYS_PAGES)*C.sysconf(C._SC_PAGE_SIZE))/
+			int64(len(taskSemaphore)), 10)
+
 	// Create a normalized thumbnail. Since we don't particularly need
-	// any complex processing, such as surrounding of metadata,
+	// any complex processing, such as surrounding metadata,
 	// simply push it through ImageMagick.
 	//
 	//  - http://www.ericbrasseur.org/gamma.html
@@ -2185,8 +2197,17 @@ func makeThumbnail(load bool, pathImage, pathThumb string) (
 	//
 	// TODO: See if we can optimize resulting WebP animations.
 	// (Do -layers optimize* apply to this format at all?)
-	cmd := exec.Command("magick", "-limit", "thread", "1", pathImage,
-		"-coalesce", "-colorspace", "RGB", "-auto-orient", "-strip",
+	cmd := exec.Command("magick", "-limit", "thread", "1",
+
+		// Do not invite the OOM killer, a particularly unpleasant guest.
+		"-limit", "memory", memoryLimit,
+
+		// ImageMagick creates files in /tmp, but that tends to be a tmpfs,
+		// which is backed by memory. The path could also be moved elsewhere:
+		// -define registry:temporary-path=/var/tmp
+		"-limit", "map", "0", "-limit", "disk", "0",
+
+		pathImage, "-coalesce", "-colorspace", "RGB", "-auto-orient", "-strip",
 		"-resize", "256x128>", "-colorspace", "sRGB",
 		"-format", "%w %h", "+write", pathThumb, "-delete", "1--1", "info:")
 
@@ -2237,7 +2258,10 @@ func cmdThumbnail(fs *flag.FlagSet, args []string) error {
 		w, h, err := makeThumbnail(*load, pathImage, pathThumb)
 		if err != nil {
 			if ee, ok := err.(*exec.ExitError); ok {
-				return string(ee.Stderr), nil
+				if message = string(ee.Stderr); message != "" {
+					return message, nil
+				}
+				return ee.String(), nil
 			}
 			return "", err
 		}
@@ -2390,14 +2414,29 @@ func cmdDhash(fs *flag.FlagSet, args []string) error {
 		}
 	}
 
-	stmt, err := db.Prepare(`UPDATE image SET dhash = ? WHERE sha1 = ?`)
+	// Commits are very IO-expensive in both WAL and non-WAL SQLite,
+	// so write this in one go. For a middle ground, we could batch the updates.
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	// Mild hack: upgrade the transaction to a write one straight away,
+	// in order to rule out deadlocks (preventable failure).
+	if _, err := tx.Exec(`END TRANSACTION;
+		BEGIN IMMEDIATE TRANSACTION`); err != nil {
+		return err
+	}
+
+	stmt, err := tx.Prepare(`UPDATE image SET dhash = ? WHERE sha1 = ?`)
 	if err != nil {
 		return err
 	}
 	defer stmt.Close()
 
 	var mu sync.Mutex
-	return parallelize(hexSHA1, func(sha1 string) (message string, err error) {
+	err = parallelize(hexSHA1, func(sha1 string) (message string, err error) {
 		hash, err := makeDhash(sha1)
 		if errors.Is(err, errIsAnimation) {
 			// Ignoring this common condition.
@@ -2411,6 +2450,10 @@ func cmdDhash(fs *flag.FlagSet, args []string) error {
 		_, err = stmt.Exec(int64(hash), sha1)
 		return "", err
 	})
+	if err != nil {
+		return err
+	}
+	return tx.Commit()
 }
 
 // --- Main --------------------------------------------------------------------
@@ -2452,6 +2495,8 @@ func usage() {
 }
 
 func main() {
+	threads := flag.Int("threads", -1, "level of parallelization")
+
 	// This implements the -h switch for us by default.
 	// The rest of the handling here closely follows what flag does internally.
 	flag.Usage = usage
@@ -2477,7 +2522,12 @@ func main() {
 		fs.PrintDefaults()
 	}
 
-	taskSemaphore = newSemaphore(runtime.NumCPU())
+	if *threads > 0 {
+		taskSemaphore = newSemaphore(*threads)
+	} else {
+		taskSemaphore = newSemaphore(runtime.NumCPU())
+	}
+
 	err := cmd.handler(fs, flag.Args()[1:])
 
 	// Note that the database object has a closing finalizer,