Clean up, add a deployment stage
All checks were successful
Alpine 3.20 Success

Errors should be handled a bit more nicely now.

The SFTP part could also be done from deploy scripts like:

 scp -i {runner.ssh.identity} \
   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
   {runner.ssh.user}@{runner.ssh.address%:*} -p {runner.ssh.address#*:}

but that is deemed way too annoying, so we do it from Go.
This commit is contained in:
Přemysl Eric Janouch 2024-12-22 09:00:02 +01:00
parent bd13053773
commit a09b11256b
Signed by: p
GPG Key ID: A0420B94F92B9493
5 changed files with 517 additions and 202 deletions

View File

@ -45,7 +45,7 @@ file present in the distribution.
All paths are currently relative to the directory you launch *acid* from.
The *notify*, *setup*, and *build* scripts are processed using Go's
The *notify*, *setup*, *build*, and *deploy* scripts are processed using Go's
_text/template_ package, and take an object describing the task,
which has the following fields:
@ -79,7 +79,8 @@ in *sh*(1) command arguments.
Runners
-------
Runners receive the following additional environment variables:
Runners and deploy scripts receive the following additional
environment variables:
*ACID_ROOT*:: The same as the base directory for configuration.
*ACID_RUNNER*:: The same as *Runner* in script templates.

627
acid.go
View File

@ -21,6 +21,7 @@ import (
"os"
"os/exec"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
@ -30,6 +31,7 @@ import (
"time"
_ "github.com/mattn/go-sqlite3"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
"gopkg.in/yaml.v3"
)
@ -96,6 +98,7 @@ func (cf *ConfigProject) AutomaticRunners() (runners []string) {
type ConfigProjectRunner struct {
Setup string `yaml:"setup"` // project setup script (SSH)
Build string `yaml:"build"` // project build script (SSH)
Deploy string `yaml:"deploy"` // project deploy script (local)
Timeout string `yaml:"timeout"` // timeout duration
}
@ -153,7 +156,8 @@ func giteaNewRequest(ctx context.Context, method, path string, body io.Reader) (
func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) {
rows, err := gDB.QueryContext(ctx, `
SELECT id, owner, repo, hash, runner,
state, detail, notified, runlog, tasklog FROM task `+query, args...)
state, detail, notified,
runlog, tasklog, deploylog FROM task `+query, args...)
if err != nil {
return nil, err
}
@ -163,7 +167,8 @@ func getTasks(ctx context.Context, query string, args ...any) ([]Task, error) {
for rows.Next() {
var t Task
err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.Hash, &t.Runner,
&t.State, &t.Detail, &t.Notified, &t.RunLog, &t.TaskLog)
&t.State, &t.Detail, &t.Notified,
&t.RunLog, &t.TaskLog, &t.DeployLog)
if err != nil {
return nil, err
}
@ -259,6 +264,10 @@ var templateTask = template.Must(template.New("tasks").Parse(`
<h2>Task log</h2>
<pre>{{printf "%s" .TaskLog}}</pre>
{{end}}
{{if .DeployLog}}
<h2>Deploy log</h2>
<pre>{{printf "%s" .DeployLog}}</pre>
{{end}}
</table>
</body>
</html>
@ -302,9 +311,12 @@ func handleTask(w http.ResponseWriter, r *http.Request) {
defer rt.RunLog.mu.Unlock()
rt.TaskLog.mu.Lock()
defer rt.TaskLog.mu.Unlock()
rt.DeployLog.mu.Lock()
defer rt.DeployLog.mu.Unlock()
task.RunLog = rt.RunLog.b
task.TaskLog = rt.TaskLog.b
task.DeployLog = rt.DeployLog.b
}()
if err := templateTask.Execute(w, &task); err != nil {
@ -816,43 +828,297 @@ func (tw *terminalWriter) Write(p []byte) (written int, err error) {
return
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// ~~~ Running task ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// RunningTask stores all data pertaining to a currently running task.
type RunningTask struct {
DB Task
Runner ConfigRunner
ProjectRunner ConfigProjectRunner
RunLog terminalWriter
TaskLog terminalWriter
RunLog terminalWriter
TaskLog terminalWriter
DeployLog terminalWriter
wd string // acid working directory
timeout time.Duration // time limit on task execution
signer ssh.Signer // SSH private key
tmplScript *ttemplate.Template // remote build script
tmplDeploy *ttemplate.Template // local deployment script
}
func executorUpdate(rt *RunningTask) error {
rt.RunLog.mu.Lock()
defer rt.RunLog.mu.Unlock()
rt.DB.RunLog = bytes.Clone(rt.RunLog.b)
if rt.DB.RunLog == nil {
rt.DB.RunLog = []byte{}
// newRunningTask prepares a task for running, without executing anything yet.
func newRunningTask(task Task) (*RunningTask, error) {
rt := &RunningTask{DB: task}
var ok bool
rt.Runner, ok = gConfig.Runners[rt.DB.Runner]
if !ok {
return nil, fmt.Errorf("unknown runner: %s", rt.DB.Runner)
}
project, ok := gConfig.Projects[rt.DB.FullName()]
if !ok {
return nil, fmt.Errorf("project configuration not found")
}
rt.ProjectRunner, ok = project.Runners[rt.DB.Runner]
if !ok {
return nil, fmt.Errorf(
"project not configured for runner %s", rt.DB.Runner)
}
rt.TaskLog.mu.Lock()
defer rt.TaskLog.mu.Unlock()
rt.DB.TaskLog = bytes.Clone(rt.TaskLog.b)
if rt.DB.TaskLog == nil {
rt.DB.TaskLog = []byte{}
var err error
if rt.wd, err = os.Getwd(); err != nil {
return nil, err
}
_, err := gDB.ExecContext(context.Background(), `UPDATE task
SET state = ?, detail = ?, notified = ?, runlog = ?, tasklog = ?
WHERE id = ?`,
rt.DB.State, rt.DB.Detail, rt.DB.Notified, rt.DB.RunLog, rt.DB.TaskLog,
rt.DB.ID)
if err == nil {
notifierAwaken()
// Lenient or not, some kind of a time limit is desirable.
rt.timeout = time.Hour
if rt.ProjectRunner.Timeout != "" {
rt.timeout, err = time.ParseDuration(rt.ProjectRunner.Timeout)
if err != nil {
return nil, fmt.Errorf("timeout: %w", err)
}
}
privateKey, err := os.ReadFile(rt.Runner.SSH.Identity)
if err != nil {
return nil, fmt.Errorf(
"cannot read SSH identity for runner %s: %w", rt.DB.Runner, err)
}
rt.signer, err = ssh.ParsePrivateKey(privateKey)
if err != nil {
return nil, fmt.Errorf(
"cannot parse SSH identity for runner %s: %w", rt.DB.Runner, err)
}
// The runner setup script may change the working directory,
// so do everything in one go. However, this approach also makes it
// difficult to distinguish project-independent runner failures.
// (For that, we can start multiple ssh.Sessions.)
//
// We could pass variables through SSH environment variables,
// which would require enabling PermitUserEnvironment in sshd_config,
// or through prepending script lines, but templates are a bit simpler.
//
// We let the runner itself clone the repository:
// - it is a more flexible in that it can test AUR packages more normally,
// - we might have to clone submodules as well.
// Otherwise, we could download a source archive from Gitea,
// and use SFTP to upload it to the runner.
rt.tmplScript, err = ttemplate.New("script").Funcs(shellFuncs).
Parse(rt.Runner.Setup + "\n" +
rt.ProjectRunner.Setup + "\n" + rt.ProjectRunner.Build)
if err != nil {
return nil, fmt.Errorf("script/build: %w", err)
}
rt.tmplDeploy, err = ttemplate.New("deploy").Funcs(shellFuncs).
Parse(rt.ProjectRunner.Deploy)
if err != nil {
return nil, fmt.Errorf("script/deploy: %w", err)
}
return rt, nil
}
// localEnv creates a process environment for locally run executables.
func (rt *RunningTask) localEnv() []string {
return append(os.Environ(),
"ACID_ROOT="+rt.wd,
"ACID_RUNNER="+rt.DB.Runner,
)
}
// update stores the running task's state in the database.
func (rt *RunningTask) update() error {
for _, i := range []struct {
tw *terminalWriter
log *[]byte
}{
{&rt.RunLog, &rt.DB.RunLog},
{&rt.TaskLog, &rt.DB.TaskLog},
{&rt.DeployLog, &rt.DB.DeployLog},
} {
i.tw.mu.Lock()
defer i.tw.mu.Unlock()
if *i.log = bytes.Clone(i.tw.b); *i.log == nil {
*i.log = []byte{}
}
}
return rt.DB.update()
}
// ~~~ Deploy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
func executorDownloadNode(sc *sftp.Client, remotePath, localPath string,
info os.FileInfo) error {
if info.IsDir() {
// Hoping that caller invokes us on parents first.
return os.MkdirAll(localPath, info.Mode().Perm())
}
src, err := sc.Open(remotePath)
if err != nil {
return fmt.Errorf("failed to open remote file %s: %w", remotePath, err)
}
defer src.Close()
dst, err := os.OpenFile(
localPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode().Perm())
if err != nil {
return fmt.Errorf("failed to create local file: %w", err)
}
defer dst.Close()
if _, err = io.Copy(dst, src); err != nil {
return fmt.Errorf("failed to copy file from remote %s to local %s: %w",
remotePath, localPath, err)
}
return nil
}
func executorDownload(client *ssh.Client, remoteRoot, localRoot string) error {
sc, err := sftp.NewClient(client)
if err != nil {
return err
}
defer sc.Close()
walker := sc.Walk(remoteRoot)
for walker.Step() {
if walker.Err() != nil {
return walker.Err()
}
relativePath, err := filepath.Rel(remoteRoot, walker.Path())
if err != nil {
return err
}
if err = executorDownloadNode(sc, walker.Path(),
filepath.Join(localRoot, relativePath), walker.Stat()); err != nil {
return err
}
}
return nil
}
func executorLocalShell() string {
if shell := os.Getenv("SHELL"); shell != "" {
return shell
}
// The os/user package doesn't store the parsed out shell field.
return "/bin/sh"
}
func executorDeploy(
ctx context.Context, client *ssh.Client, rt *RunningTask) error {
script := bytes.NewBuffer(nil)
if err := rt.tmplDeploy.Execute(script, &rt.DB); err != nil {
return &executorError{"Deploy template failed", err}
}
// Thus the deployment directory must exist iff the script is not empty.
if script.Len() == 0 {
return nil
}
// We expect the files to be moved elsewhere on the filesystem,
// and they may get very large, so avoid /tmp.
//
// See also: https://systemd.io/TEMPORARY_DIRECTORIES/
tmp := os.Getenv("TMPDIR")
if tmp == "" {
tmp = "/var/tmp"
}
dir := filepath.Join(tmp, "acid-deploy")
if err := os.RemoveAll(dir); err != nil {
return err
}
if err := os.Mkdir(dir, 0755); err != nil {
return err
}
// The passed remoteRoot is relative to sc.Getwd.
if err := executorDownload(client, "acid-deploy", dir); err != nil {
return err
}
cmd := exec.CommandContext(ctx, executorLocalShell(), "-c", script.String())
cmd.Env = rt.localEnv()
cmd.Dir = dir
cmd.Stdout = &rt.DeployLog
cmd.Stderr = &rt.DeployLog
return cmd.Run()
}
// ~~~ Build ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
func executorBuild(
ctx context.Context, client *ssh.Client, rt *RunningTask) error {
// This is here to fail early, though logically it is misplaced.
script := bytes.NewBuffer(nil)
if err := rt.tmplScript.Execute(script, &rt.DB); err != nil {
return &executorError{"Script template failed", err}
}
session, err := client.NewSession()
if err != nil {
return &executorError{"SSH failure", err}
}
defer session.Close()
modes := ssh.TerminalModes{ssh.ECHO: 0}
if err := session.RequestPty("dumb", 24, 80, modes); err != nil {
return &executorError{"SSH failure", err}
}
log.Printf("task %d for %s: connected\n", rt.DB.ID, rt.DB.FullName())
session.Stdout = &rt.TaskLog
session.Stderr = &rt.TaskLog
// Although passing the script directly takes away the option to specify
// a particular shell (barring here-documents!), it is simple and reliable.
//
// Passing the script over Stdin to sh tended to end up with commands
// eating the script during processing, and resulted in a hang,
// because closing the Stdin does not result in remote processes
// getting a stream of EOF.
//
// Piping the script into `cat | sh` while appending a ^D to the end of it
// appeared to work, but it seems likely that commands might still steal
// script bytes from the cat program if they choose to read from the tty
// and the script is longer than the buffer.
chSession := make(chan error, 1)
go func() {
chSession <- session.Run(script.String())
close(chSession)
}()
select {
case <-ctx.Done():
// Either shutdown, or early runner termination.
// The runner is not supposed to finish before the session.
err = context.Cause(ctx)
case err = <-chSession:
// Killing a runner may perfectly well trigger this first,
// in particular when it's on the same machine.
}
return err
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// executorError describes a taskStateError.
type executorError struct {
Detail string
Err error
}
func (e *executorError) Unwrap() error { return e.Err }
func (e *executorError) Error() string {
return fmt.Sprintf("%s: %s", e.Detail, e.Err)
}
func executorConnect(
ctx context.Context, config *ssh.ClientConfig, address string) (
*ssh.Client, error) {
@ -894,122 +1160,57 @@ func executorConnect(
time.Sleep(1 * time.Second)
continue
}
return ssh.NewClient(sc, chans, reqs), nil
}
}
func executorRunTask(ctx context.Context, task Task) error {
rt := &RunningTask{DB: task}
var ok bool
rt.Runner, ok = gConfig.Runners[rt.DB.Runner]
if !ok {
return fmt.Errorf("unknown runner: %s", rt.DB.Runner)
}
project, ok := gConfig.Projects[rt.DB.FullName()]
if !ok {
return fmt.Errorf("project configuration not found")
}
rt.ProjectRunner, ok = project.Runners[rt.DB.Runner]
if !ok {
return fmt.Errorf(
"project not configured for runner %s", rt.DB.Runner)
}
wd, err := os.Getwd()
rt, err := newRunningTask(task)
if err != nil {
return err
task.State, task.Detail = taskStateError, "Misconfigured"
task.Notified = 0
task.RunLog = []byte(err.Error())
task.TaskLog = []byte{}
task.DeployLog = []byte{}
return task.update()
}
// The runner setup script may change the working directory,
// so do everything in one go. However, this approach also makes it
// difficult to distinguish project-independent runner failures.
// (For that, we can start multiple ssh.Sessions.)
//
// We could pass variables through SSH environment variables,
// which would require enabling PermitUserEnvironment in sshd_config,
// or through prepending script lines, but templates are a bit simpler.
//
// We let the runner itself clone the repository:
// - it is a more flexible in that it can test AUR packages more normally,
// - we might have to clone submodules as well.
// Otherwise, we could download a source archive from Gitea,
// and use SFTP to upload it to the runner.
tmplScript, err := ttemplate.New("script").Funcs(shellFuncs).
Parse(rt.Runner.Setup + "\n" +
rt.ProjectRunner.Setup + "\n" + rt.ProjectRunner.Build)
if err != nil {
return fmt.Errorf("script: %w", err)
}
// Lenient or not, some kind of a time limit is desirable.
timeout := time.Hour
if rt.ProjectRunner.Timeout != "" {
timeout, err = time.ParseDuration(rt.ProjectRunner.Timeout)
if err != nil {
return fmt.Errorf("timeout: %w", err)
}
}
ctx, cancelTimeout := context.WithTimeout(ctx, timeout)
ctx, cancelTimeout := context.WithTimeout(ctx, rt.timeout)
defer cancelTimeout()
privateKey, err := os.ReadFile(rt.Runner.SSH.Identity)
if err != nil {
return fmt.Errorf(
"cannot read SSH identity for runner %s: %w", rt.DB.Runner, err)
}
signer, err := ssh.ParsePrivateKey(privateKey)
if err != nil {
return fmt.Errorf(
"cannot parse SSH identity for runner %s: %w", rt.DB.Runner, err)
}
defer func() {
// RunningTasks can be concurrently accessed by HTTP handlers.
locked := func(f func()) {
gRunningMutex.Lock()
defer gRunningMutex.Unlock()
delete(gRunning, rt.DB.ID)
}()
func() {
gRunningMutex.Lock()
defer gRunningMutex.Unlock()
f()
}
locked(func() {
rt.DB.State, rt.DB.Detail = taskStateRunning, ""
rt.DB.Notified = 0
rt.DB.RunLog = []byte{}
rt.DB.TaskLog = []byte{}
rt.DB.DeployLog = []byte{}
gRunning[rt.DB.ID] = rt
}()
if err := executorUpdate(rt); err != nil {
})
defer locked(func() {
delete(gRunning, rt.DB.ID)
})
if err := rt.update(); err != nil {
return fmt.Errorf("SQL: %w", err)
}
// Errors happening while trying to write an error are unfortunate,
// but not important enough to abort entirely.
setError := func(detail string) {
gRunningMutex.Lock()
defer gRunningMutex.Unlock()
rt.DB.State, rt.DB.Detail = taskStateError, detail
if err := executorUpdate(rt); err != nil {
if err := rt.update(); err != nil {
log.Printf("error: task %d for %s: SQL: %s",
rt.DB.ID, rt.DB.FullName(), err)
}
}
script := bytes.NewBuffer(nil)
if err := tmplScript.Execute(script, &rt.DB); err != nil {
setError("Script template failed")
return err
}
cmd := exec.CommandContext(ctx, rt.Runner.Run)
cmd.Env = append(
os.Environ(),
"ACID_ROOT="+wd,
"ACID_RUNNER="+rt.DB.Runner,
)
cmd.Env = rt.localEnv()
// Pushing the runner into a new process group that can be killed at once
// with all its children isn't bullet-proof, it messes with job control
@ -1033,7 +1234,8 @@ func executorRunTask(ctx context.Context, task Task) error {
cmd.Stdout = &rt.RunLog
cmd.Stderr = &rt.RunLog
if err := cmd.Start(); err != nil {
setError("Runner failed to start")
fmt.Fprintf(&rt.TaskLog, "%s\n", err)
locked(func() { setError("Runner failed to start") })
return err
}
@ -1059,78 +1261,61 @@ func executorRunTask(ctx context.Context, task Task) error {
client, err := executorConnect(ctxRunner, &ssh.ClientConfig{
User: rt.Runner.SSH.User,
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
Auth: []ssh.AuthMethod{ssh.PublicKeys(rt.signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}, rt.Runner.SSH.Address)
if err != nil {
fmt.Fprintf(&rt.TaskLog, "%s\n", err)
setError("SSH failure")
locked(func() { setError("SSH failure") })
return err
}
defer client.Close()
session, err := client.NewSession()
var (
ee1 *ssh.ExitError
ee2 *executorError
)
err = executorBuild(ctxRunner, client, rt)
if err != nil {
fmt.Fprintf(&rt.TaskLog, "%s\n", err)
setError("SSH failure")
return err
}
defer session.Close()
modes := ssh.TerminalModes{ssh.ECHO: 0}
if err := session.RequestPty("dumb", 24, 80, modes); err != nil {
fmt.Fprintf(&rt.TaskLog, "%s\n", err)
setError("SSH failure")
return err
locked(func() {
if errors.As(err, &ee1) {
rt.DB.State, rt.DB.Detail = taskStateFailed, "Scripts failed"
fmt.Fprintf(&rt.TaskLog, "\n%s\n", err)
} else if errors.As(err, &ee2) {
rt.DB.State, rt.DB.Detail = taskStateError, ee2.Detail
fmt.Fprintf(&rt.TaskLog, "\n%s\n", ee2.Err)
} else {
rt.DB.State, rt.DB.Detail = taskStateError, ""
fmt.Fprintf(&rt.TaskLog, "\n%s\n", err)
}
})
return rt.update()
}
log.Printf("task %d for %s: connected\n", rt.DB.ID, rt.DB.FullName())
session.Stdout = &rt.TaskLog
session.Stderr = &rt.TaskLog
// Although passing the script directly takes away the option to specify
// a particular shell (barring here-documents!), it is simple and reliable.
//
// Passing the script over Stdin to sh tended to end up with commands
// eating the script during processing, and resulted in a hang,
// because closing the Stdin does not result in remote processes
// getting a stream of EOF.
//
// Piping the script into `cat | sh` while appending a ^D to the end of it
// appeared to work, but it seems likely that commands might still steal
// script bytes from the cat program if they choose to read from the tty
// and the script is longer than the buffer.
chSession := make(chan error, 1)
// This is so that it doesn't stay hanging within the sftp package,
// which uses context.Background() everywhere.
go func() {
chSession <- session.Run(script.String())
close(chSession)
<-ctxRunner.Done()
client.Close()
}()
select {
case <-ctxRunner.Done():
// Either shutdown, or early runner termination.
// The runner is not supposed to finish before the session.
err = context.Cause(ctxRunner)
case err = <-chSession:
// Killing a runner may perfectly well trigger this first,
// in particular when it's on the same machine.
}
gRunningMutex.Lock()
defer gRunningMutex.Unlock()
var ee *ssh.ExitError
if err == nil {
rt.DB.State, rt.DB.Detail = taskStateSuccess, ""
} else if errors.As(err, &ee) {
rt.DB.State, rt.DB.Detail = taskStateFailed, "Scripts failed"
fmt.Fprintf(&rt.TaskLog, "\n%s\n", err)
} else {
rt.DB.State, rt.DB.Detail = taskStateError, ""
fmt.Fprintf(&rt.TaskLog, "\n%s\n", err)
}
return executorUpdate(rt)
err = executorDeploy(ctxRunner, client, rt)
locked(func() {
if err == nil {
rt.DB.State, rt.DB.Detail = taskStateSuccess, ""
} else if errors.As(err, &ee1) {
rt.DB.State, rt.DB.Detail = taskStateFailed, "Deployment failed"
fmt.Fprintf(&rt.DeployLog, "\n%s\n", err)
} else if errors.As(err, &ee2) {
rt.DB.State, rt.DB.Detail = taskStateError, ee2.Detail
fmt.Fprintf(&rt.DeployLog, "\n%s\n", ee2.Err)
} else {
rt.DB.State, rt.DB.Detail = taskStateError, ""
fmt.Fprintf(&rt.DeployLog, "\n%s\n", err)
}
})
return rt.update()
}
func executorRun(ctx context.Context) error {
@ -1208,11 +1393,12 @@ type Task struct {
Hash string
Runner string
State taskState
Detail string
Notified int64
RunLog []byte
TaskLog []byte
State taskState
Detail string
Notified int64
RunLog []byte
TaskLog []byte
DeployLog []byte
}
func (t *Task) FullName() string { return t.Owner + "/" + t.Repo }
@ -1242,26 +1428,58 @@ func (t *Task) CloneURL() string {
return fmt.Sprintf("%s/%s/%s.git", gConfig.Gitea, t.Owner, t.Repo)
}
const schemaSQL = `
func (t *Task) update() error {
_, err := gDB.ExecContext(context.Background(), `UPDATE task
SET state = ?, detail = ?, notified = ?,
runlog = ?, tasklog = ?, deploylog = ? WHERE id = ?`,
t.State, t.Detail, t.Notified,
t.RunLog, t.TaskLog, t.DeployLog, t.ID)
if err == nil {
notifierAwaken()
}
return err
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
const initializeSQL = `
PRAGMA application_id = 0x61636964; -- "acid" in big endian
CREATE TABLE IF NOT EXISTS task(
id INTEGER NOT NULL, -- unique ID
id INTEGER NOT NULL, -- unique ID
owner TEXT NOT NULL, -- Gitea username
repo TEXT NOT NULL, -- Gitea repository name
hash TEXT NOT NULL, -- commit hash
runner TEXT NOT NULL, -- the runner to use
owner TEXT NOT NULL, -- Gitea username
repo TEXT NOT NULL, -- Gitea repository name
hash TEXT NOT NULL, -- commit hash
runner TEXT NOT NULL, -- the runner to use
state INTEGER NOT NULL DEFAULT 0, -- task state
detail TEXT NOT NULL DEFAULT '', -- task state detail
notified INTEGER NOT NULL DEFAULT 0, -- Gitea knows the state
runlog BLOB NOT NULL DEFAULT x'', -- combined task runner output
tasklog BLOB NOT NULL DEFAULT x'', -- combined task SSH output
state INTEGER NOT NULL DEFAULT 0, -- task state
detail TEXT NOT NULL DEFAULT '', -- task state detail
notified INTEGER NOT NULL DEFAULT 0, -- Gitea knows the state
runlog BLOB NOT NULL DEFAULT x'', -- combined task runner output
tasklog BLOB NOT NULL DEFAULT x'', -- combined task SSH output
deploylog BLOB NOT NULL DEFAULT x'', -- deployment output
PRIMARY KEY (id)
) STRICT;
`
func openDB(path string) error {
func dbEnsureColumn(tx *sql.Tx, table, column, definition string) error {
var count int64
if err := tx.QueryRow(
`SELECT count(*) FROM pragma_table_info(?) WHERE name = ?`,
table, column).Scan(&count); err != nil {
return err
} else if count == 1 {
return nil
}
_, err := tx.Exec(
`ALTER TABLE ` + table + ` ADD COLUMN ` + column + ` ` + definition)
return err
}
func dbOpen(path string) error {
var err error
gDB, err = sql.Open("sqlite3",
"file:"+path+"?_foreign_keys=1&_busy_timeout=1000")
@ -1269,10 +1487,43 @@ func openDB(path string) error {
return err
}
_, err = gDB.Exec(schemaSQL)
return err
tx, err := gDB.BeginTx(context.Background(), nil)
if err != nil {
return err
}
defer tx.Rollback()
var version int64
if err = tx.QueryRow(`PRAGMA user_version`).Scan(&version); err != nil {
return err
}
switch version {
case 0:
if _, err = tx.Exec(initializeSQL); err != nil {
return err
}
// We had not initially set a database schema version,
// so we're stuck checking this column even on new databases.
if err = dbEnsureColumn(tx,
`task`, `deploylog`, `BLOB NOT NULL DEFAULT x''`); err != nil {
return err
}
break
case 1:
// The next migration goes here, remember to increment the number below.
}
if _, err = tx.Exec(
`PRAGMA user_version = ` + strconv.Itoa(1)); err != nil {
return err
}
return tx.Commit()
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// callRPC forwards command line commands to a running server.
func callRPC(args []string) error {
body, err := json.Marshal(args)
@ -1334,7 +1585,7 @@ func main() {
return
}
if err := openDB(gConfig.DB); err != nil {
if err := dbOpen(gConfig.DB); err != nil {
log.Fatalln(err)
}
defer gDB.Close()

View File

@ -61,7 +61,13 @@ projects:
# Project build script.
build: |
echo Computing line count...
find . -not -path '*/.*' -type f -print0 | xargs -0 cat | wc -l
mkdir ~/acid-deploy
find . -not -path '*/.*' -type f -print0 | xargs -0 cat | wc -l \
> ~/acid-deploy/count
# Project deployment script (runs locally in a temporary directory).
deploy: |
cat count
# Time limit in time.ParseDuration format.
# The default of one hour should suffice.

10
go.mod
View File

@ -3,9 +3,13 @@ module janouch.name/acid
go 1.22.0
require (
github.com/mattn/go-sqlite3 v1.14.22
golang.org/x/crypto v0.21.0
github.com/mattn/go-sqlite3 v1.14.24
github.com/pkg/sftp v1.13.7
golang.org/x/crypto v0.31.0
gopkg.in/yaml.v3 v3.0.1
)
require golang.org/x/sys v0.18.0 // indirect
require (
github.com/kr/fs v0.1.0 // indirect
golang.org/x/sys v0.28.0 // indirect
)

69
go.sum
View File

@ -1,12 +1,65 @@
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=