speatzle/morffix
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

move ffmpeg command to seperate table

Browse source
This commit is contained in:
Samuel Lorch 2024-06-23 22:47:50 +02:00
parent 72dfd2408f
commit cdeb266e38
20 changed files with 245 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
constants/constants.go
View file

@ -16,6 +16,7 @@ const LIBRARY_TEMPLATE_NAME = "library.tmpl"
const MESSAGE_TEMPLATE_NAME = "message.tmpl"
const TASKS_TEMPLATE_NAME = "tasks.tmpl"
const TASK_TEMPLATE_NAME = "task.tmpl"
const FFMPEG_COMMANDS_TEMPLATE_NAME = "ffmpeg_commands.tmpl"
const FORM_FILE_KEY = "file"

2
migrations/000008_alter_files_table_updated_at.down.sql
View file

@ -1,2 +1,2 @@
ALTER TABLE files
DROP updated_at;
DROP IF EXISTS updated_at;

1
migrations/000012_create_ffmpeg_commands_table.down.sql Normal file
View file

@ -0,0 +1 @@
DROP TABLE IF EXISTS ffmpeg_commands;

6
migrations/000012_create_ffmpeg_commands_table.up.sql Normal file
View file

@ -0,0 +1,6 @@
CREATE TABLE IF NOT EXISTS ffmpeg_commands(
id bigserial PRIMARY KEY,
name VARCHAR (50) NOT NULL,
data JSONB
);

1
migrations/000013_insert_healthcheck.down.sql Normal file
View file

@ -0,0 +1 @@
DELETE FROM ffmpeg_commands where id = 1;

1
migrations/000013_insert_healthcheck.up.sql Normal file
View file

@ -0,0 +1 @@
INSERT INTO ffmpeg_commands (name, data) VALUES ('healthcheck_command','{"args": [{"flag": "-stats", "value": ""}, {"flag": "-v", "value": "error"}, {"flag": "-xerror", "value": ""}], "input_files": [{"args": null, "path": "input.mkv"}], "output_files": [{"args": [{"flag": "-f", "value": "null"}, {"flag": "-max_muxing_queue_size", "value": "9999"}], "path": "output.mkv"}]}');

2
migrations/000014_alter_task_table_ffmpeg_command.down.sql Normal file
View file

@ -0,0 +1,2 @@
ALTER TABLE tasks
DROP IF EXISTS ffmpeg_command;

2
migrations/000014_alter_task_table_ffmpeg_command.up.sql Normal file
View file

@ -0,0 +1,2 @@
ALTER TABLE tasks
ADD ffmpeg_command_id bigint REFERENCES ffmpeg_commands(id) NOT NULL DEFAULT 1;

79
server/ffmpeg_command.go Normal file
View file

@ -0,0 +1,79 @@
package server
import (
"bytes"
"fmt"
"log/slog"
"net/http"
"git.lastassault.de/speatzle/morffix/constants"
"github.com/jackc/pgx/v5"
)
type FfmpegCommandsData struct {
FfmpegCommands []FfmpegCommand
}
type FfmpegCommand struct {
ID int `db:"id"`
Name string `db:"name"`
Data string `db:"data"`
}
func handleFfmpegCommands(w http.ResponseWriter, r *http.Request) {
data := FfmpegCommandsData{}
if r.Method == "POST" {
err := createFfmpegCommand(r)
if err != nil {
slog.ErrorContext(r.Context(), "Create FfmpegCommand", "err", err)
http.Error(w, "Error Create FfmpegCommand: "+err.Error(), http.StatusInternalServerError)
return
}
}
rows, err := db.Query(r.Context(), "SELECT id, name, data FROM ffmpeg_commands")
if err != nil {
slog.ErrorContext(r.Context(), "Query ffmpegCommands", "err", err)
http.Error(w, "Error Query ffmpegCommands: "+err.Error(), http.StatusInternalServerError)
return
}
ffmpegCommands, err := pgx.CollectRows[FfmpegCommand](rows, pgx.RowToStructByName[FfmpegCommand])
if err != nil {
slog.ErrorContext(r.Context(), "Collect Rows", "err", err)
http.Error(w, "Error Query ffmpegCommands: "+err.Error(), http.StatusInternalServerError)
return
}
data.FfmpegCommands = ffmpegCommands
buf := bytes.Buffer{}
err = templates.ExecuteTemplate(&buf, constants.FFMPEG_COMMANDS_TEMPLATE_NAME, data)
if err != nil {
slog.ErrorContext(r.Context(), "Executing ffmpegCommands Template", "err", err)
http.Error(w, "Error Executing Template: "+err.Error(), http.StatusInternalServerError)
return
}
_, err = w.Write(buf.Bytes())
if err != nil {
slog.ErrorContext(r.Context(), "Writing http Response", "err", err)
}
}
func createFfmpegCommand(r *http.Request) error {
err := r.ParseForm()
if err != nil {
return fmt.Errorf("Parseing Form: %w", err)
}
name := r.FormValue("name")
data := r.FormValue("data")
slog.Info("Got FfmpegCommand Create", "name", name, "data", data)
_, err = db.Exec(r.Context(), "INSERT INTO ffmpegCommands (name, data) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", name, data)
if err != nil {
return fmt.Errorf("Inserting FfmpegCommand: %w", err)
}
return nil
}

2
server/server.go
View file

@ -100,6 +100,8 @@ func Start(_conf config.Config, tmplFS embed.FS, staticFS embed.FS, migrationsFS
mux.HandleFunc("/scan/{id}", handleScan)
mux.HandleFunc("/libraries/{id}", handleLibrary)
mux.HandleFunc("/libraries", handleLibraries)
mux.HandleFunc("/ffmpeg_commands", handleFfmpegCommands)
mux.HandleFunc("/", handleIndex)
server := &http.Server{

128
server/task.go
View file

@ -16,9 +16,10 @@ import (
)
type TasksData struct {
Libraries []Library
Tasks []TaskDisplay
Stats TaskStats
Libraries []Library
FfmpegCommands []FfmpegCommand
Tasks []TaskDisplay
Stats TaskStats
}
type TaskStats struct {
@ -34,23 +35,25 @@ type TaskStats struct {
}
type TaskDisplay struct {
ID int `db:"id"`
Library int `db:"library"`
Worker *string `db:"worker"`
Type int `db:"type"`
Status string `db:"status"`
File string `db:"file"`
UpdatedAt string `db:"updated_at"`
ID int `db:"id"`
Library int `db:"library"`
Worker *string `db:"worker"`
Type int `db:"type"`
FfmpegCommand *string `db:"ffmpeg_command"`
Status string `db:"status"`
File string `db:"file"`
UpdatedAt string `db:"updated_at"`
}
type TaskDB struct {
ID int `db:"id"`
Library int `db:"library"`
Worker *string `db:"worker"`
Type int `db:"type"`
Status constants.TaskStatus `db:"status"`
File string `db:"file"`
UpdatedAt time.Time `db:"updated_at"`
ID int `db:"id"`
Library int `db:"library"`
Worker *string `db:"worker"`
Type int `db:"type"`
FfmpegCommand *string `db:"ffmpeg_command"`
Status constants.TaskStatus `db:"status"`
File string `db:"file"`
UpdatedAt time.Time `db:"updated_at"`
}
func handleTasks(w http.ResponseWriter, r *http.Request) {
@ -92,12 +95,26 @@ func handleTasks(w http.ResponseWriter, r *http.Request) {
libraries, err := pgx.CollectRows[Library](rows, pgx.RowToStructByName[Library])
if err != nil {
slog.ErrorContext(r.Context(), "Collect Rows", "err", err)
http.Error(w, "Error Query Libraries: "+err.Error(), http.StatusInternalServerError)
http.Error(w, "Error Collect Libraries: "+err.Error(), http.StatusInternalServerError)
return
}
data.Libraries = libraries
rows, err = db.Query(r.Context(), "SELECT t.id AS id, l.id AS library, t.worker_id AS worker, t.type AS type, t.status AS status, f.path AS file, t.updated_at AS updated_at FROM tasks t INNER JOIN files f ON f.id = t.file_id INNER JOIN libraries l ON l.id = f.library_id ORDER BY CASE t.type WHEN 3 THEN -1 ELSE t.type END, t.id")
rows, err = db.Query(r.Context(), "SELECT id, name, data FROM ffmpeg_commands")
if err != nil {
slog.ErrorContext(r.Context(), "Query Ffmpeg Commands", "err", err)
http.Error(w, "Error Ffmpeg Commands: "+err.Error(), http.StatusInternalServerError)
return
}
ffmpegCommands, err := pgx.CollectRows[FfmpegCommand](rows, pgx.RowToStructByName[FfmpegCommand])
if err != nil {
slog.ErrorContext(r.Context(), "Collect Ffmpeg Commands", "err", err)
http.Error(w, "Error Collect Ffmpeg Commands: "+err.Error(), http.StatusInternalServerError)
return
}
data.FfmpegCommands = ffmpegCommands
rows, err = db.Query(r.Context(), "SELECT t.id AS id, l.id AS library, t.worker_id AS worker, t.type AS type, fc.name AS ffmpeg_command, t.status AS status, f.path AS file, t.updated_at AS updated_at FROM tasks t INNER JOIN files f ON f.id = t.file_id INNER JOIN libraries l ON l.id = f.library_id INNER JOIN ffmpeg_commands fc ON fc.id = t.ffmpeg_command_id ORDER BY CASE t.type WHEN 3 THEN -1 ELSE t.type END, t.id")
if err != nil {
slog.ErrorContext(r.Context(), "Query Tasks", "err", err)
http.Error(w, "Error Query Tasks: "+err.Error(), http.StatusInternalServerError)
@ -111,13 +128,14 @@ func handleTasks(w http.ResponseWriter, r *http.Request) {
}
for i := range tasks {
data.Tasks = append(data.Tasks, TaskDisplay{
ID: tasks[i].ID,
Library: tasks[i].Library,
Worker: tasks[i].Worker,
Type: tasks[i].Type,
File: tasks[i].File,
Status: tasks[i].Status.String(),
UpdatedAt: tasks[i].UpdatedAt.Format(time.DateTime),
ID: tasks[i].ID,
Library: tasks[i].Library,
Worker: tasks[i].Worker,
Type: tasks[i].Type,
FfmpegCommand: tasks[i].FfmpegCommand,
File: tasks[i].File,
Status: tasks[i].Status.String(),
UpdatedAt: tasks[i].UpdatedAt.Format(time.DateTime),
})
}
@ -198,23 +216,25 @@ func createTask(ctx context.Context, r *http.Request) error {
defer tx.Rollback(ctx)
var data any
if typ == constants.TASK_TYPE_HEALTHCHECK {
/*
if typ == constants.TASK_TYPE_HEALTHCHECK {
// ffmpeg.exe -stats -v error -i "in.mp4" -f null -max_muxing_queue_size 9999 "out.mp4"
data = types.HealthCheckData{Command: types.FFmpegCommand{
Args: []types.Arg{{Flag: "-stats"}, {Flag: "-v", Value: "error"}, {Flag: "-xerror"}},
InputFiles: []types.File{{Path: "input.mkv"}},
OutputFiles: []types.File{{Path: "output.mkv", Arguments: []types.Arg{{Flag: "-f", Value: "null"}, {Flag: "-max_muxing_queue_size", Value: "9999"}}}},
}}
} else if typ == constants.TASK_TYPE_TRANSCODE {
data = types.TranscodeData{Command: types.FFmpegCommand{
Args: []types.Arg{{Flag: "-stats"}, {Flag: "-v", Value: "error"}, {Flag: "-xerror"}},
InputFiles: []types.File{{Path: "input.mkv"}},
OutputFiles: []types.File{{Path: "output.mkv", Arguments: []types.Arg{{Flag: "-map", Value: "0"}, {Flag: "-c", Value: "copy"}, {Flag: "-c:a", Value: "aac"}, {Flag: "-c:v", Value: "libsvtav1"}, {Flag: "-crf", Value: "35"}, {Flag: "-preset", Value: "8"}, {Flag: "-max_muxing_queue_size", Value: "9999"}}}},
}}
} else {
return fmt.Errorf("Unkown Task Type: %v", typ)
}
// ffmpeg.exe -stats -v error -i "in.mp4" -f null -max_muxing_queue_size 9999 "out.mp4"
data = types.HealthCheckData{Command: types.FFmpegCommand{
Args: []types.Arg{{Flag: "-stats"}, {Flag: "-v", Value: "error"}, {Flag: "-xerror"}},
InputFiles: []types.File{{Path: "input.mkv"}},
OutputFiles: []types.File{{Path: "output.mkv", Arguments: []types.Arg{{Flag: "-f", Value: "null"}, {Flag: "-max_muxing_queue_size", Value: "9999"}}}},
}}
} else if typ == constants.TASK_TYPE_TRANSCODE {
data = types.TranscodeData{Command: types.FFmpegCommand{
Args: []types.Arg{{Flag: "-stats"}, {Flag: "-v", Value: "error"}, {Flag: "-xerror"}},
InputFiles: []types.File{{Path: "input.mkv"}},
OutputFiles: []types.File{{Path: "output.mkv", Arguments: []types.Arg{{Flag: "-map", Value: "0"}, {Flag: "-c", Value: "copy"}, {Flag: "-c:a", Value: "aac"}, {Flag: "-c:v", Value: "libsvtav1"}, {Flag: "-crf", Value: "35"}, {Flag: "-preset", Value: "8"}, {Flag: "-max_muxing_queue_size", Value: "9999"}}}},
}}
} else {
return fmt.Errorf("Unkown Task Type: %v", typ)
}
*/
for _, file := range files {
_, err = db.Exec(r.Context(), "INSERT INTO tasks (file_id, type, status, data) VALUES ($1,$2,$3,$4)", file.ID, typ, constants.TASK_STATUS_QUEUED, data)
@ -232,15 +252,16 @@ func createTask(ctx context.Context, r *http.Request) error {
}
type QueuedTask struct {
ID int
Type int
FileID int `json:"file_id"`
FileMD5 []byte `json:"file_md5" db:"md5"`
Data json.RawMessage
ID int
Type int
FileID int `json:"file_id"`
FileMD5 []byte `json:"file_md5" db:"md5"`
Data json.RawMessage
FfmpegCommand types.FFmpegCommand `json:"ffmpeg_command" db:"ffmpeg_command"`
}
func assignQueuedTasks(ctx context.Context) error {
rows, err := db.Query(ctx, "SELECT t.id as id, t.type as type, t.file_id as file_id, f.md5 as md5, t.data as data FROM tasks t INNER JOIN files f ON f.id = t.file_id WHERE t.status = $1", constants.TASK_STATUS_QUEUED)
rows, err := db.Query(ctx, "SELECT t.id as id, t.type as type, t.file_id as file_id, f.md5 as md5, t.data as data, fc.data as ffmpeg_command FROM tasks t INNER JOIN files f ON f.id = t.file_id INNER JOIN ffmpeg_commands fc ON fc.id = t.ffmpeg_command_id WHERE t.status = $1", constants.TASK_STATUS_QUEUED)
if err != nil {
return fmt.Errorf("Query Queued Tasks: %w", err)
}
@ -288,11 +309,12 @@ func assignQueuedTasks(ctx context.Context) error {
}
taskStart := types.TaskStart{
ID: queuedTasks[lastAssigned].ID,
Type: queuedTasks[lastAssigned].Type,
FileID: queuedTasks[lastAssigned].FileID,
FileMD5: queuedTasks[lastAssigned].FileMD5,
Data: queuedTasks[lastAssigned].Data,
ID: queuedTasks[lastAssigned].ID,
Type: queuedTasks[lastAssigned].Type,
FileID: queuedTasks[lastAssigned].FileID,
FileMD5: queuedTasks[lastAssigned].FileMD5,
Data: queuedTasks[lastAssigned].Data,
FfmpegCommand: queuedTasks[lastAssigned].FfmpegCommand,
}
_, err = rpcServer.Call(ctx, Workers[i].Conn, "task-start", taskStart, nil)

8
task/healthcheck.go
View file

@ -20,12 +20,12 @@ func RunHealthCheck(conf config.Config, t *types.Task, data types.HealthCheckDat
path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("%v-%v.mkv", t.ID, t.FileID))
// Set ffmpeg input path
if len(data.Command.InputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no input files")
if len(t.FfmpegCommand.InputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no input files", "command", t.FfmpegCommand)
return
}
data.Command.InputFiles[0].Path = path
t.FfmpegCommand.InputFiles[0].Path = path
// TODO cleanup file when done
defer func() {
@ -43,7 +43,7 @@ func RunHealthCheck(conf config.Config, t *types.Task, data types.HealthCheckDat
return
}
err = runFfmpegCommand(ctx, l, conf, data.Command)
err = runFfmpegCommand(ctx, l, conf, t.FfmpegCommand)
if err != nil {
l.ErrorContext(ctx, "FFmpeg Failed", "err", err)
return

9
task/task.go
View file

@ -18,10 +18,11 @@ func StartTask(conf config.Config, data types.TaskStart) error {
defer taskMutex.Unlock()
tasks[data.ID] = &types.Task{
ID: data.ID,
Type: data.Type,
FileID: data.FileID,
FileMD5: data.FileMD5,
ID: data.ID,
Type: data.Type,
FileID: data.FileID,
FileMD5: data.FileMD5,
FfmpegCommand: data.FfmpegCommand,
}
switch data.Type {

14
task/transcode.go
View file

@ -21,20 +21,20 @@ func RunTranscode(conf config.Config, t *types.Task, data types.TranscodeData) {
dst_path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("dst-%v-%v.mkv", t.ID, t.FileID))
// Set ffmpeg input path
if len(data.Command.InputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no input files")
if len(t.FfmpegCommand.InputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no input files", "command", t.FfmpegCommand)
return
}
data.Command.InputFiles[0].Path = src_path
t.FfmpegCommand.InputFiles[0].Path = src_path
// Set ffmpeg output path
if len(data.Command.OutputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no output files")
if len(t.FfmpegCommand.OutputFiles) == 0 {
l.ErrorContext(ctx, "FFmpeg Command has no output files", "command", t.FfmpegCommand)
return
}
data.Command.OutputFiles[0].Path = dst_path
t.FfmpegCommand.OutputFiles[0].Path = dst_path
// TODO cleanup file when done
defer func() {
@ -62,7 +62,7 @@ func RunTranscode(conf config.Config, t *types.Task, data types.TranscodeData) {
}
}()
err = runFfmpegCommand(ctx, l, conf, data.Command)
err = runFfmpegCommand(ctx, l, conf, t.FfmpegCommand)
if err != nil {
l.ErrorContext(ctx, "FFmpeg Failed", "err", err)
return

32
tmpl/ffmpeg_commands.tmpl Normal file
View file

@ -0,0 +1,32 @@
{{template "head"}}
<h2>New FFmpeg Command</h2>
<form method="POST">
<label>Name:</label>
<input type="text" name="name">
<label>Data:</label>
<input type="text" name="data">
<input type="submit" value="Submit">
</form>
<h2>FFmpeg Commands:</h2>
<table>
<tr>
<th>ID</th>
<th>Name</th>
<th>Data</th>
</tr>
{{range $t := .FfmpegCommands}}
<tr onclick="window.location='/ffmpeg_commands/{{ $t.ID }}';">
<td>
{{ $t.ID }}
</td>
<td>
{{ $t.Name }}
</td>
<td>
{{ $t.Data }}
</td>
</tr>
{{end}}
</table>
{{template "tail"}}

1
tmpl/navbar.tmpl
View file

@ -3,5 +3,6 @@
<a class="button" href="/">Morffix</a>
<a class="button" href="/libraries">Libraries</a>
<a class="button" href="/tasks">Tasks</a>
<a class="button" href="/ffmpeg_commands">FFmpeg Commands</a>
</nav>
{{end}}

14
tmpl/tasks.tmpl
View file

@ -16,8 +16,14 @@
</select>
<label for="type">Type</label>
<select id="type" name="type">
<option value="0">Health Check</option>
<option value="1">Transcode</option>
<option value="0">Health Check</option>
<option value="1">Transcode</option>
</select>
<label for="ffmpeg_command">FFmpeg Command:</label>
<select id="ffmpeg_command" name="ffmpeg_command">
{{range $l := .FfmpegCommands}}
<option value="{{$l.ID}}">{{$l.Name}}</option>
{{end}}
</select>
<input type="submit" value="Submit">
</form>
@ -42,6 +48,7 @@ Total: {{.Stats.TotalCount}}
<th>Library</th>
<th>Worker</th>
<th>Type</th>
<th>FFmpeg Command</th>
<th>Status</th>
<th>File</th>
<th>Updated At</th>
@ -60,6 +67,9 @@ Total: {{.Stats.TotalCount}}
<td>
{{ $t.Type }}
</td>
<td>
{{ $t.FfmpegCommand }}
</td>
<td>
{{ $t.Status }}
</td>

1
types/healtcheck.go
View file

@ -1,5 +1,4 @@
package types
type HealthCheckData struct {
Command FFmpegCommand `json:"command"`
}

24
types/task.go
View file

@ -7,20 +7,22 @@ import (
)
type TaskStart struct {
ID int `json:"id"`
FileID int `json:"file_id"`
FileMD5 []byte `json:"file_md5"`
Type int `json:"type"`
Data json.RawMessage
ID int `json:"id"`
FileID int `json:"file_id"`
FileMD5 []byte `json:"file_md5"`
Type int `json:"type"`
Data json.RawMessage
FfmpegCommand FFmpegCommand `json:"ffmpeg_command"`
}
type Task struct {
ID int `json:"id"`
FileID int `json:"file_id"`
FileMD5 []byte `json:"md5"`
Type int `json:"type"`
Status constants.TaskStatus `json:"status"`
Log []string `json:"log"`
ID int `json:"id"`
FileID int `json:"file_id"`
FileMD5 []byte `json:"md5"`
Type int `json:"type"`
Status constants.TaskStatus `json:"status"`
FfmpegCommand FFmpegCommand `json:"ffmpeg_command"`
Log []string `json:"log"`
}
type TaskStatus struct {

1
types/transcode.go
View file

@ -1,5 +1,4 @@
package types
type TranscodeData struct {
Command FFmpegCommand `json:"command"`
}