speatzle/morffix
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

Ask For Task Status Seperatly and logs incrementally

Browse source
This commit is contained in:
speatzle 2024-05-09 19:09:48 +02:00
parent d185784cd1
commit 12e83aa8d2
4 changed files with 73 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

55
server/worker.go
View file

@ -9,6 +9,7 @@ import (
"git.lastassault.de/speatzle/morffix/rpc"
"git.lastassault.de/speatzle/morffix/types"
"github.com/jackc/pgx/v5"
"git.lastassault.de/speatzle/morffix/constants"
@ -173,41 +174,53 @@ func updateWorkerTaskStatus(ctx context.Context) {
for uuid := range Workers {
if Workers[uuid].Connected {
w := Workers[uuid]
wg.Add(1)
go func() {
defer wg.Done()
var status types.TaskStatus
_, err := rpcServer.Call(ctx, w.Conn, "task-status", nil, &status)
if err != nil {
slog.ErrorContext(ctx, "Error Getting Task Status", "err", err)
// TODO Mark Worker / Tasks as Unknown?
return
}
rows, err := db.Query(ctx, "SELECT id, COALESCE(CARDINALITY(log),0) as log_offset FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4)", uuid, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING)
if err != nil {
slog.ErrorContext(ctx, "Error Getting Tasks for Worker", "err", err, "worker_id", uuid)
return
}
for _, t := range status.Tasks {
// TODO check if this Task was even assigned to this Worker
taskStatusRequests, err := pgx.CollectRows[types.TaskStatusRequest](rows, pgx.RowToStructByName[types.TaskStatusRequest])
if err != nil {
slog.ErrorContext(ctx, "Collect Task Status Rows", "err", err)
return
}
_, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = $3 WHERE id = $1", t.ID, t.Status, t.Log)
for _, tsr := range taskStatusRequests {
wg.Add(1)
go func() {
defer wg.Done()
var ts types.TaskStatus
_, err := rpcServer.Call(ctx, w.Conn, "task-status", tsr, &ts)
if err != nil {
slog.ErrorContext(ctx, "Error Getting Task Status", "err", err)
// TODO Mark Task as Unknown?
return
}
_, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", ts.Task.ID, ts.Task.Status, ts.Task.Log)
if err != nil {
slog.ErrorContext(ctx, "Error Updating Task Status", "err", err)
continue
return
}
// Tell Worker to Delete Finished Tasks
if t.Status == constants.TASK_STATUS_FAILED || t.Status == constants.TASK_STATUS_SUCCESS {
_, err := rpcServer.Call(ctx, w.Conn, "task-delete", t.ID, nil)
if ts.Task.Status == constants.TASK_STATUS_FAILED || ts.Task.Status == constants.TASK_STATUS_SUCCESS {
_, err := rpcServer.Call(ctx, w.Conn, "task-delete", ts.Task.ID, nil)
if err != nil {
slog.ErrorContext(ctx, "Error Deleting Finished Task From Worker", "err", err)
continue
return
}
}
}
// TODO Set Task Status to QUEUED For Unfinished Tasks which where assigned to this worker but where not in the status response
}()
}()
}
// TODO Handle tasks with status unkown assigned to this worker
} else {
// Set Task Status to Unknown for Unfinished Tasks which where assigned to this not connected worker
// Set Task Status to Unknown for Unfinished Tasks which where assigned to this not connected worker, right now they just get stuck
}
}
}()