Compare commits
No commits in common. "e895beadb7ce5309d6ffeec1090de5e72e8e34db" and "537b48dc221ea5d375ac8f61603787bac49efc3c" have entirely different histories.
e895beadb7
...
537b48dc22
11
.gitignore
vendored
11
.gitignore
vendored
@ -1,11 +0,0 @@
|
|||||||
/gallery
|
|
||||||
/initialize.go
|
|
||||||
/public/mithril.js
|
|
||||||
|
|
||||||
/gallery.cflags
|
|
||||||
/gallery.config
|
|
||||||
/gallery.creator
|
|
||||||
/gallery.creator.user
|
|
||||||
/gallery.cxxflags
|
|
||||||
/gallery.files
|
|
||||||
/gallery.includes
|
|
70
main.go
70
main.go
@ -1286,9 +1286,6 @@ type syncContext struct {
|
|||||||
stmtDisposeSub *sql.Stmt
|
stmtDisposeSub *sql.Stmt
|
||||||
stmtDisposeAll *sql.Stmt
|
stmtDisposeAll *sql.Stmt
|
||||||
|
|
||||||
// exclude specifies filesystem paths that should be seen as missing.
|
|
||||||
exclude *regexp.Regexp
|
|
||||||
|
|
||||||
// linked tracks which image hashes we've checked so far in the run.
|
// linked tracks which image hashes we've checked so far in the run.
|
||||||
linked map[string]struct{}
|
linked map[string]struct{}
|
||||||
}
|
}
|
||||||
@ -1697,12 +1694,6 @@ func syncDirectory(c *syncContext, dbParent int64, fsPath string) error {
|
|||||||
fs = nil
|
fs = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.exclude != nil {
|
|
||||||
fs = slices.DeleteFunc(fs, func(f syncFile) bool {
|
|
||||||
return c.exclude.MatchString(filepath.Join(fsPath, f.fsName))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert differences to a form more convenient for processing.
|
// Convert differences to a form more convenient for processing.
|
||||||
iDB, iFS, pairs := 0, 0, []syncPair{}
|
iDB, iFS, pairs := 0, 0, []syncPair{}
|
||||||
for iDB < len(db) && iFS < len(fs) {
|
for iDB < len(db) && iFS < len(fs) {
|
||||||
@ -1878,21 +1869,9 @@ const disposeCTE = `WITH RECURSIVE
|
|||||||
HAVING count = total
|
HAVING count = total
|
||||||
)`
|
)`
|
||||||
|
|
||||||
type excludeRE struct{ re *regexp.Regexp }
|
|
||||||
|
|
||||||
func (re *excludeRE) String() string { return fmt.Sprintf("%v", re.re) }
|
|
||||||
|
|
||||||
func (re *excludeRE) Set(value string) error {
|
|
||||||
var err error
|
|
||||||
re.re, err = regexp.Compile(value)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// cmdSync ensures the given (sub)roots are accurately reflected
|
// cmdSync ensures the given (sub)roots are accurately reflected
|
||||||
// in the database.
|
// in the database.
|
||||||
func cmdSync(fs *flag.FlagSet, args []string) error {
|
func cmdSync(fs *flag.FlagSet, args []string) error {
|
||||||
var exclude excludeRE
|
|
||||||
fs.Var(&exclude, "exclude", "exclude paths matching regular expression")
|
|
||||||
fullpaths := fs.Bool("fullpaths", false, "don't basename arguments")
|
fullpaths := fs.Bool("fullpaths", false, "don't basename arguments")
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return err
|
return err
|
||||||
@ -1930,7 +1909,7 @@ func cmdSync(fs *flag.FlagSet, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
c := syncContext{ctx: ctx, tx: tx, pb: newProgressBar(-1),
|
c := syncContext{ctx: ctx, tx: tx, pb: newProgressBar(-1),
|
||||||
exclude: exclude.re, linked: make(map[string]struct{})}
|
linked: make(map[string]struct{})}
|
||||||
defer c.pb.Stop()
|
defer c.pb.Stop()
|
||||||
|
|
||||||
if c.stmtOrphan, err = c.tx.Prepare(disposeCTE + `
|
if c.stmtOrphan, err = c.tx.Prepare(disposeCTE + `
|
||||||
@ -2148,54 +2127,36 @@ func collectFileListing(root string) (paths []string, err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkFiles(gc bool,
|
func checkFiles(root, suffix string, hashes []string) (bool, []string, error) {
|
||||||
root, suffix string, hashes []string) (bool, []string, error) {
|
|
||||||
db := hashesToFileListing(root, suffix, hashes)
|
db := hashesToFileListing(root, suffix, hashes)
|
||||||
fs, err := collectFileListing(root)
|
fs, err := collectFileListing(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, err
|
return false, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// There are two legitimate cases of FS-only database files:
|
iDB, iFS, ok, intersection := 0, 0, true, []string{}
|
||||||
// 1. There is no code to unlink images at all
|
|
||||||
// (although sync should create orphan records for everything).
|
|
||||||
// 2. thumbnail: failures may result in an unreferenced garbage image.
|
|
||||||
ok := true
|
|
||||||
onlyDB := func(path string) {
|
|
||||||
ok = false
|
|
||||||
fmt.Printf("only in DB: %s\n", path)
|
|
||||||
}
|
|
||||||
onlyFS := func(path string) {
|
|
||||||
if !gc {
|
|
||||||
ok = false
|
|
||||||
fmt.Printf("only in FS: %s\n", path)
|
|
||||||
} else if err := os.Remove(path); err != nil {
|
|
||||||
ok = false
|
|
||||||
fmt.Printf("only in FS (removing failed): %s: %s\n", path, err)
|
|
||||||
} else {
|
|
||||||
fmt.Printf("only in FS (removing): %s\n", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
iDB, iFS, intersection := 0, 0, []string{}
|
|
||||||
for iDB < len(db) && iFS < len(fs) {
|
for iDB < len(db) && iFS < len(fs) {
|
||||||
if db[iDB] == fs[iFS] {
|
if db[iDB] == fs[iFS] {
|
||||||
intersection = append(intersection, db[iDB])
|
intersection = append(intersection, db[iDB])
|
||||||
iDB++
|
iDB++
|
||||||
iFS++
|
iFS++
|
||||||
} else if db[iDB] < fs[iFS] {
|
} else if db[iDB] < fs[iFS] {
|
||||||
onlyDB(db[iDB])
|
ok = false
|
||||||
|
fmt.Printf("only in DB: %s\n", db[iDB])
|
||||||
iDB++
|
iDB++
|
||||||
} else {
|
} else {
|
||||||
onlyFS(fs[iFS])
|
ok = false
|
||||||
|
fmt.Printf("only in FS: %s\n", fs[iFS])
|
||||||
iFS++
|
iFS++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, path := range db[iDB:] {
|
for _, path := range db[iDB:] {
|
||||||
onlyDB(path)
|
ok = false
|
||||||
|
fmt.Printf("only in DB: %s\n", path)
|
||||||
}
|
}
|
||||||
for _, path := range fs[iFS:] {
|
for _, path := range fs[iFS:] {
|
||||||
onlyFS(path)
|
ok = false
|
||||||
|
fmt.Printf("only in FS: %s\n", path)
|
||||||
}
|
}
|
||||||
return ok, intersection, nil
|
return ok, intersection, nil
|
||||||
}
|
}
|
||||||
@ -2243,7 +2204,6 @@ func checkHashes(paths []string) (bool, error) {
|
|||||||
// cmdCheck carries out various database consistency checks.
|
// cmdCheck carries out various database consistency checks.
|
||||||
func cmdCheck(fs *flag.FlagSet, args []string) error {
|
func cmdCheck(fs *flag.FlagSet, args []string) error {
|
||||||
full := fs.Bool("full", false, "verify image hashes")
|
full := fs.Bool("full", false, "verify image hashes")
|
||||||
gc := fs.Bool("gc", false, "garbage collect database files")
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -2280,13 +2240,13 @@ func cmdCheck(fs *flag.FlagSet, args []string) error {
|
|||||||
|
|
||||||
// This somewhat duplicates {image,thumb}Path().
|
// This somewhat duplicates {image,thumb}Path().
|
||||||
log.Println("checking SQL against filesystem")
|
log.Println("checking SQL against filesystem")
|
||||||
okImages, intersection, err := checkFiles(*gc,
|
okImages, intersection, err := checkFiles(
|
||||||
filepath.Join(galleryDirectory, nameOfImageRoot), "", allSHA1)
|
filepath.Join(galleryDirectory, nameOfImageRoot), "", allSHA1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
okThumbs, _, err := checkFiles(*gc,
|
okThumbs, _, err := checkFiles(
|
||||||
filepath.Join(galleryDirectory, nameOfThumbRoot), ".webp", thumbSHA1)
|
filepath.Join(galleryDirectory, nameOfThumbRoot), ".webp", thumbSHA1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -2295,11 +2255,11 @@ func cmdCheck(fs *flag.FlagSet, args []string) error {
|
|||||||
ok = false
|
ok = false
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Println("checking for dead symlinks (should become orphans on sync)")
|
log.Println("checking for dead symlinks")
|
||||||
for _, path := range intersection {
|
for _, path := range intersection {
|
||||||
if _, err := os.Stat(path); err != nil {
|
if _, err := os.Stat(path); err != nil {
|
||||||
ok = false
|
ok = false
|
||||||
fmt.Printf("%s: %s\n", path, err.(*os.PathError).Unwrap())
|
fmt.Printf("%s: %s\n", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
7
test.sh
7
test.sh
@ -16,9 +16,6 @@ sha1duplicate=$sha1
|
|||||||
cp $input/Test/dhash.png \
|
cp $input/Test/dhash.png \
|
||||||
$input/Test/multiple-paths.png
|
$input/Test/multiple-paths.png
|
||||||
|
|
||||||
gen -seed 15 -size 256x256 plasma:fractal \
|
|
||||||
$input/Test/excluded.png
|
|
||||||
|
|
||||||
gen -seed 20 -size 160x128 plasma:fractal \
|
gen -seed 20 -size 160x128 plasma:fractal \
|
||||||
-bordercolor transparent -border 64 \
|
-bordercolor transparent -border 64 \
|
||||||
$input/Test/transparent-wide.png
|
$input/Test/transparent-wide.png
|
||||||
@ -39,7 +36,7 @@ gen $input/Test/animation-small.gif \
|
|||||||
$input/Test/video.mp4
|
$input/Test/video.mp4
|
||||||
|
|
||||||
./gallery init $target
|
./gallery init $target
|
||||||
./gallery sync -exclude '/excluded[.]' $target $input "$@"
|
./gallery sync $target $input "$@"
|
||||||
./gallery thumbnail $target
|
./gallery thumbnail $target
|
||||||
./gallery dhash $target
|
./gallery dhash $target
|
||||||
./gallery tag $target test "Test space" <<-END
|
./gallery tag $target test "Test space" <<-END
|
||||||
@ -50,7 +47,7 @@ END
|
|||||||
|
|
||||||
# TODO: Test all the various possible sync transitions.
|
# TODO: Test all the various possible sync transitions.
|
||||||
mv $input/Test $input/Plasma
|
mv $input/Test $input/Plasma
|
||||||
./gallery sync -exclude '/excluded[.]' $target $input
|
./gallery sync $target $input
|
||||||
|
|
||||||
./gallery web $target :8080 &
|
./gallery web $target :8080 &
|
||||||
web=$!
|
web=$!
|
||||||
|
Loading…
x
Reference in New Issue
Block a user