wip

constants/constants.go

@@ -87,6 +87,7 @@ const (
 	FILE_STATUS_MISSING
 	FILE_STATUS_EXISTS
 	FILE_STATUS_CHANGED
+	FILE_STATUS_NEW
 )
 
 func (s FileStatus) String() string {
@@ -99,6 +100,8 @@ func (s FileStatus) String() string {
 		return "Exists"
 	case FILE_STATUS_CHANGED:
 		return "Changed"
+	case FILE_STATUS_NEW:
+		return "New"
 	default:
 		return fmt.Sprintf("%d", int(s))
 	}

migrations/000015_alter_files_table_hash.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE files
+ALTER COLUMN hash bigint SET NOT NULL;

migrations/000015_alter_files_table_hash.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE files
+ALTER COLUMN hash bigint DROP NOT NULL;

migrations/000016_alter_workers_table_queue_enable.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE workers
+DROP IF EXISTS  queue_enable;

migrations/000016_alter_workers_table_queue_enable.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE workers
+ADD queue_enable boolean NOT NULL DEFAULT false;

server/index.go

@@ -16,8 +16,10 @@ type IndexData struct {
 }
 
 type IndexWorker struct {
+	ID string
 	Worker
-	Status *types.WorkerStatus
+	Status      *types.WorkerStatus
+	QueueEnable bool
 }
 
 func handleIndex(w http.ResponseWriter, r *http.Request) {
@@ -39,11 +41,13 @@ func handleIndex(w http.ResponseWriter, r *http.Request) {
 				}
 				slog.InfoContext(r.Context(), "Got Worker Status", "id", i, "status", status)
 				data.Workers = append(data.Workers, IndexWorker{
+					ID:     i,
 					Worker: *Workers[i],
 					Status: &status,
 				})
 			} else {
 				data.Workers = append(data.Workers, IndexWorker{
+					ID:     i,
 					Worker: *Workers[i],
 					Status: nil,
 				})

server/queue.go

@@ -0,0 +1,40 @@
+package server
+
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+)
+
+func HandleSetQueueEnable(w http.ResponseWriter, r *http.Request) {
+	err := r.ParseForm()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Parseing Form: %v", err), http.StatusBadRequest)
+		return
+	}
+	enable := r.FormValue("enable")
+	if enable != "true" && enable != "false" {
+		http.Error(w, "Enable must be true or false", http.StatusBadRequest)
+		return
+	}
+
+	worker := r.FormValue("worker")
+
+	slog.Info("Got set Queue Enable", "enable", enable, "worker", worker)
+
+	if worker == "all" {
+		_, err = db.Exec(r.Context(), "UPDATE workers SET queue_enable = $1", enable)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Setting Worker Queue Enable: %v", err), http.StatusInternalServerError)
+			return
+		}
+	} else {
+		_, err = db.Exec(r.Context(), "UPDATE workers SET queue_enable = $1 where id = $2", enable, worker)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Setting Worker Queue Enable: %v", err), http.StatusInternalServerError)
+			return
+		}
+	}
+
+	http.Redirect(w, r, r.Header.Get("Referer"), http.StatusFound)
+}

server/scan.go

@@ -3,10 +3,8 @@ package server
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"errors"
 	"fmt"
-	"io"
 	"log/slog"
 	"net/http"
 	"os"
@@ -31,8 +29,6 @@ func handleScan(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	full := r.FormValue("full") == "on"
-
 	var name string
 	var path string
 	var enabled bool
@@ -44,7 +40,7 @@ func handleScan(w http.ResponseWriter, r *http.Request) {
 	}
 
 	scanCtx := context.Background()
-	go scan(scanCtx, id, full)
+	go scan(scanCtx, id)
 
 	message := "Scan Started"
 
@@ -79,7 +75,7 @@ func scanStatus(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func scan(ctx context.Context, id string, full bool) {
+func scan(ctx context.Context, id string) {
 	slog.InfoContext(ctx, "Starting Scan", "id", id)
 
 	// TODO Scan settings:
@@ -150,35 +146,36 @@ func scan(ctx context.Context, id string, full bool) {
 				slog.InfoContext(ctx, "Skipping non video file", "path", fullPath)
 				return nil
 			}
-			slog.InfoContext(ctx, "Hashing File", "path", fullPath, "size", info.Size())
+			/*
+				slog.InfoContext(ctx, "Hashing File", "path", fullPath, "size", info.Size())
 
-			file, err := os.Open(fullPath)
+					file, err := os.Open(fullPath)
-			if err != nil {
+					if err != nil {
-				return fmt.Errorf("Opening File: %w", err)
+						return fmt.Errorf("Opening File: %w", err)
-			}
+					}
 
-			hash := md5.New()
+						hash := md5.New()
-			if _, err := io.Copy(hash, file); err != nil {
+						if _, err := io.Copy(hash, file); err != nil {
-				return fmt.Errorf("Reading File: %w", err)
+							return fmt.Errorf("Reading File: %w", err)
-			}
+						}
-			newMD5 := hash.Sum(nil)
+						newMD5 := hash.Sum(nil)
-
-			slog.InfoContext(ctx, "File MD5", "path", fullPath, "size", info.Size(), "md5", newMD5)
 
+						slog.InfoContext(ctx, "File MD5", "path", fullPath, "size", info.Size(), "md5", newMD5)
+			*/
 			fPath, err := filepath.Rel(lpath, fullPath)
 			if err != nil {
 				return fmt.Errorf("Getting Relative Path: %w", err)
 			}
 
 			var fileID int
-			var oldMD5 []byte
+			var size uint
 			var health constants.FileHealth
-			err = tx.QueryRow(ctx, "SELECT id, md5, health FROM files WHERE library_id = $1 AND path = $2", id, fPath).Scan(&fileID, &oldMD5, &health)
+			err = tx.QueryRow(ctx, "SELECT id, size, health FROM files WHERE library_id = $1 AND path = $2", id, fPath).Scan(&fileID, &size, &health)
 			if errors.Is(err, pgx.ErrNoRows) {
 				// File Does not Exist Yet
 
 				slog.InfoContext(ctx, "File is New", "path", fullPath)
-				_, err = tx.Exec(ctx, "INSERT INTO files (library_id, path, size, status, health, md5) VALUES ($1, $2, $3, $4, $5, $6)", id, fPath, info.Size(), constants.FILE_STATUS_EXISTS, constants.FILE_HEALTH_UNKNOWN, newMD5)
+				_, err = tx.Exec(ctx, "INSERT INTO files (library_id, path, size, status, health) VALUES ($1, $2, $3, $4, $5)", id, fPath, info.Size(), constants.FILE_STATUS_NEW, constants.FILE_HEALTH_UNKNOWN)
 				if err != nil {
 					return fmt.Errorf("Add New File to DB: %w", err)
 				}
@@ -187,12 +184,14 @@ func scan(ctx context.Context, id string, full bool) {
 				return fmt.Errorf("Getting File: %w", err)
 			}
 
-			if slices.Compare[[]byte](newMD5, oldMD5) != 0 {
+			/*
-				// File has changed on disk so reset health
+				if slices.Compare[[]byte](newMD5, oldMD5) != 0 {
-				health = constants.FILE_HEALTH_UNKNOWN
+					// File has changed on disk so reset health
-			}
+					health = constants.FILE_HEALTH_UNKNOWN
+				}
+			*/
 			// File Exists so update Size, status and hash
-			_, err = tx.Exec(ctx, "UPDATE files SET size = $2, status = $3, health = $4, md5 = $5 WHERE id = $1", fileID, info.Size(), constants.FILE_STATUS_EXISTS, health, newMD5)
+			_, err = tx.Exec(ctx, "UPDATE files SET size = $2, status = $3, health = $4 WHERE id = $1", fileID, info.Size(), constants.FILE_STATUS_EXISTS, health)
 			if err != nil {
 				return fmt.Errorf("Updating File in DB: %w", err)
 			}

server/server.go

@@ -101,6 +101,7 @@ func Start(_conf config.Config, tmplFS embed.FS, staticFS embed.FS, migrationsFS
 	mux.HandleFunc("/libraries/{id}", handleLibrary)
 	mux.HandleFunc("/libraries", handleLibraries)
 	mux.HandleFunc("/ffmpeg_commands", handleFfmpegCommands)
+	mux.HandleFunc("/queue_enable", HandleSetQueueEnable)
 
 	mux.HandleFunc("/", handleIndex)
 

server/task.go

@@ -332,7 +332,18 @@ type QueuedTask struct {
 	FfmpegCommand types.FFmpegCommand `json:"ffmpeg_command" db:"ffmpeg_command"`
 }
 
+var assignRunning = false
+
 func assignQueuedTasks(ctx context.Context) error {
+	if assignRunning {
+		return nil
+	}
+
+	assignRunning = true
+	defer func() {
+		assignRunning = false
+	}()
+
 	rows, err := db.Query(ctx, "SELECT t.id as id, t.type as type, t.file_id as file_id, f.md5 as md5, t.data as data, fc.data as ffmpeg_command FROM tasks t INNER JOIN files f ON f.id = t.file_id INNER JOIN ffmpeg_commands fc ON fc.id = t.ffmpeg_command_id WHERE t.status = $1", constants.TASK_STATUS_QUEUED)
 	if err != nil {
 		return fmt.Errorf("Query Queued Tasks: %w", err)
@@ -360,13 +371,23 @@ func assignQueuedTasks(ctx context.Context) error {
 			return nil
 		}
 		if Workers[i].Connected {
+			var queueEnable bool
+			err := db.QueryRow(ctx, "SELECT queue_enable FROM workers WHERE id = $1", i).Scan(&queueEnable)
+			if err != nil {
+				return fmt.Errorf("Error Querying Worker Queue Enable: %w", err)
+			}
+			if !queueEnable {
+				slog.DebugContext(ctx, "Skipping Worker since Queueing is disabled", "worker_id", i)
+				continue
+			}
+
 			var count int
-			err := db.QueryRow(ctx, "SELECT COUNT(*) FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", i, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING).Scan(&count)
+			err = db.QueryRow(ctx, "SELECT COUNT(*) FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", i, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING).Scan(&count)
 			if err != nil {
 				return fmt.Errorf("Error Querying Worker Task Count: %w", err)
 			}
 
-			slog.Info("Assigning Queued Tasks Worker", "worker", i, "count", count)
+			slog.Debug("Assigning Queued Tasks Worker", "worker", i, "count", count)
 
 			// Allow for Multiple Tasks at once in the future
 			if count < 1 {
@@ -395,7 +416,7 @@ func assignQueuedTasks(ctx context.Context) error {
 						// Task was started previously but something went wrong and we are out of sync
 						slog.WarnContext(ctx, "Task is apparently already Running on this Worker, thats bad", "task_id", taskStart.ID, "worker", Workers[i].Name)
 
-						_, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_ASSIGNED, []string{fmt.Sprintf("%v MASTER: Task Start, Task Already Running!", time.Now())})
+						_, err = tx.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_ASSIGNED, []string{fmt.Sprintf("%v MASTER: Task Start, Task Already Running!", time.Now())})
 						if err != nil {
 							return fmt.Errorf("Updating Task status during already running error: %w", err)
 						}
@@ -403,14 +424,14 @@ func assignQueuedTasks(ctx context.Context) error {
 						// We really don't know whats going on, might be slow response, oom, disk full or a bug
 						slog.WarnContext(ctx, "Task start Timed Out", "task_id", taskStart.ID, "worker", Workers[i].Name)
 
-						_, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Task Start RPC Call Timed Out!", time.Now())})
+						_, err = tx.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Task Start RPC Call Timed Out!", time.Now())})
 						if err != nil {
 							return fmt.Errorf("Updating Unknown Task Status due to Timeout while starting Task: %w", err)
 						}
 					} else {
 						slog.ErrorContext(ctx, "Task start Error", "task_id", taskStart.ID, "worker", Workers[i].Name)
 
-						_, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Task Start Error: %v", time.Now(), err.Error())})
+						_, err = tx.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Task Start Error: %v", time.Now(), err.Error())})
 						if err != nil {
 							return fmt.Errorf("Updating Unknown Task Status due to Error while starting Task: %w", err)
 						}

server/worker.go

@@ -129,7 +129,7 @@ func readMessage(ctx context.Context, c *websocket.Conn) error {
 	if err != nil {
 		return err
 	}
-	slog.InfoContext(ctx, "Got Websocket Message", "type", t.String(), "data", data)
+	slog.DebugContext(ctx, "Got Websocket Message", "type", t.String(), "data", data)
 	rpcServer.HandleMessage(ctx, c, data)
 	return nil
 }

tmpl/index.tmpl

@@ -15,6 +15,22 @@
         {{end}}
     {{end}}
 </div>
+<h2>Set Queue Enable</h2>
+<form method="POST" action= "/queue_enable">
+    <label for="worker">Worker:</label>
+    <select id="worker" name="worker">
+        <option value="all">All</option>
+        {{range $w := .Workers}}
+        <option value="{{$w.ID}}">{{$w.Name}}</option>
+        {{end}}
+    </select>
+    <label for="enable">Enable</label>
+    <select id="enable" name="enable">
+        <option value="true">True</option>
+        <option value="false">False</option>
+    </select>
+    <input type="submit" value="Submit">
+</form>
 <h2>Workers</h2>
 <div class="workers">
     {{range $w := .Workers}}

worker/worker.go

@@ -116,7 +116,7 @@ func readMessage(ctx context.Context, c *websocket.Conn) error {
 	if err != nil {
 		return err
 	}
-	slog.InfoContext(ctx, "Got Websocket Message", "type", t.String(), "data", data)
+	slog.DebugContext(ctx, "Got Websocket Message", "type", t.String(), "data", data)
 	rpcServer.HandleMessage(ctx, c, data)
 	return nil
 }