From 182f3819b3a92d5924b785d855b2b7522eda2475 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:00:49 +0200 Subject: [PATCH 01/30] initial ci --- .forgejo/workflows/build.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .forgejo/workflows/build.yaml diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml new file mode 100644 index 0000000..5a7be2b --- /dev/null +++ b/.forgejo/workflows/build.yaml @@ -0,0 +1,14 @@ +on: [push] +jobs: + test: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - run: go build + - uses: actions/upload-artifact@v4 + with: + name: morffix + path: morffix \ No newline at end of file From 1acf6183dda81f8529ebddb8a16b565d0d21be9f Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:04:50 +0200 Subject: [PATCH 02/30] fix artifacts --- .forgejo/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 5a7be2b..a2f32e7 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -8,7 +8,7 @@ jobs: with: go-version: '1.22' - run: go build - - uses: actions/upload-artifact@v4 + - uses: forgejo/upload-artifact@v4 with: name: morffix path: morffix \ No newline at end of file From 08ecab2e5672b62260d3678a4cd7d3977523703b Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:11:00 +0200 Subject: [PATCH 03/30] attempt 2 --- .forgejo/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index a2f32e7..365b947 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -8,7 +8,7 @@ jobs: with: go-version: '1.22' - run: go build - - uses: forgejo/upload-artifact@v4 + - uses: forgejo/upload-artifact@v3 with: name: morffix path: morffix \ No newline at end of file From 017ff526edae96adbe8ed9a6c523eb31cc07371a Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:38:09 +0200 Subject: [PATCH 04/30] add release --- .forgejo/workflows/build.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 365b947..fb102db 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -11,4 +11,10 @@ jobs: - uses: forgejo/upload-artifact@v3 with: name: morffix - path: morffix \ No newline at end of file + path: morffix + - uses: actions/forgejo-release@v1 + with: + direction: upload + url: https://git.soontm.de + release-dir: dist/release + release-notes: "New Release" \ No newline at end of file From dfc9e0a1bc9fc41dddcd95a45aa89299ed8de89a Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:40:52 +0200 Subject: [PATCH 05/30] fix workflow --- .forgejo/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index fb102db..e25a20c 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -14,7 +14,7 @@ jobs: path: morffix - uses: actions/forgejo-release@v1 with: - direction: upload - url: https://git.soontm.de + direction: upload + url: https://git.soontm.de release-dir: dist/release release-notes: "New Release" \ No newline at end of file From 6a07db76ca83a2ddd0e3d2dcb51303a89b451206 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:43:25 +0200 Subject: [PATCH 06/30] upload correct file --- .forgejo/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index e25a20c..b22982f 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -16,5 +16,5 @@ jobs: with: direction: upload url: https://git.soontm.de - release-dir: dist/release + release-dir: morffix release-notes: "New Release" \ No newline at end of file From e4fc76e6b94697f34853b1a835cf3a21c4b326be Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:48:16 +0200 Subject: [PATCH 07/30] use upload with arch fix --- .forgejo/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index b22982f..7642273 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -12,7 +12,7 @@ jobs: with: name: morffix path: morffix - - uses: actions/forgejo-release@v1 + - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 with: direction: upload url: https://git.soontm.de From e9c5e4f088200f3286603ea5ba0e0aafdf2422f5 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 22:51:53 +0200 Subject: [PATCH 08/30] add token --- .forgejo/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 7642273..22a313f 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -17,4 +17,5 @@ jobs: direction: upload url: https://git.soontm.de release-dir: morffix - release-notes: "New Release" \ No newline at end of file + release-notes: "New Release" + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 3895e17090eee82207c514d81b86d0c764e3ab1e Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:04:41 +0200 Subject: [PATCH 09/30] add release --- .forgejo/workflows/build.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 22a313f..b962290 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -12,10 +12,22 @@ jobs: with: name: morffix path: morffix + release: + runs-on: docker + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - run: go build - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 with: direction: upload url: https://git.soontm.de - release-dir: morffix + release-dir: ./ release-notes: "New Release" - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} + + + \ No newline at end of file From 0fe9e7943bc72ef2610440662bbc7b86db1cfee6 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:07:08 +0200 Subject: [PATCH 10/30] always run release --- .forgejo/workflows/build.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index b962290..e0f9b5a 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -1,20 +1,7 @@ on: [push] jobs: - test: - runs-on: docker - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.22' - - run: go build - - uses: forgejo/upload-artifact@v3 - with: - name: morffix - path: morffix release: runs-on: docker - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 From a98a172b79d4c31be8935c1f51567a01ac00f038 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:20:09 +0200 Subject: [PATCH 11/30] revert --- .forgejo/workflows/build.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index e0f9b5a..b962290 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -1,7 +1,20 @@ on: [push] jobs: + test: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - run: go build + - uses: forgejo/upload-artifact@v3 + with: + name: morffix + path: morffix release: runs-on: docker + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 From a29d7ec013c41391ca438280e6e87127e308959d Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:25:21 +0200 Subject: [PATCH 12/30] add override --- .forgejo/workflows/build.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index b962290..7ba142b 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -1,20 +1,7 @@ on: [push] jobs: - test: - runs-on: docker - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.22' - - run: go build - - uses: forgejo/upload-artifact@v3 - with: - name: morffix - path: morffix release: runs-on: docker - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 @@ -26,7 +13,9 @@ jobs: direction: upload url: https://git.soontm.de release-dir: ./ - release-notes: "New Release" + release-notes: "Latest Release" + override: true + tag: latest token: ${{ secrets.GITHUB_TOKEN }} From 471a545ba377db8d38ed8d39442ffcff17da71d7 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:37:06 +0200 Subject: [PATCH 13/30] add latest tag --- .forgejo/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 7ba142b..18f486e 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -8,6 +8,7 @@ jobs: with: go-version: '1.22' - run: go build + - uses: https://github.com/EndBug/latest-tag@v1.6.2 - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 with: direction: upload From 659b9ddaca5bf106d376c8b7472829dfae1e5428 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:39:52 +0200 Subject: [PATCH 14/30] lets go back to manual tags --- .forgejo/workflows/build.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 18f486e..b962290 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -1,6 +1,6 @@ on: [push] jobs: - release: + test: runs-on: docker steps: - uses: actions/checkout@v4 @@ -8,15 +8,25 @@ jobs: with: go-version: '1.22' - run: go build - - uses: https://github.com/EndBug/latest-tag@v1.6.2 + - uses: forgejo/upload-artifact@v3 + with: + name: morffix + path: morffix + release: + runs-on: docker + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - run: go build - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 with: direction: upload url: https://git.soontm.de release-dir: ./ - release-notes: "Latest Release" - override: true - tag: latest + release-notes: "New Release" token: ${{ secrets.GITHUB_TOKEN }} From 927d4b695a6a0a64054819e09630916d0109fa80 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:52:20 +0200 Subject: [PATCH 15/30] change release dir --- .forgejo/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index b962290..4696dfb 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -21,11 +21,13 @@ jobs: with: go-version: '1.22' - run: go build + - run: mkdir out + - run: cp morffix out/morffix - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 with: direction: upload url: https://git.soontm.de - release-dir: ./ + release-dir: out release-notes: "New Release" token: ${{ secrets.GITHUB_TOKEN }} From 4a2d3f03576d1b7df5f3ad736aeb624a2b9574ac Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:54:25 +0200 Subject: [PATCH 16/30] add override --- .forgejo/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 4696dfb..da58767 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -30,6 +30,7 @@ jobs: release-dir: out release-notes: "New Release" token: ${{ secrets.GITHUB_TOKEN }} + override: true \ No newline at end of file From 9e8c5545cc516e2256f07c888d4cb5d4db3cdecf Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:56:23 +0200 Subject: [PATCH 17/30] always run release --- .forgejo/workflows/build.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index da58767..e0035ad 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -1,20 +1,7 @@ on: [push] jobs: - test: - runs-on: docker - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.22' - - run: go build - - uses: forgejo/upload-artifact@v3 - with: - name: morffix - path: morffix release: runs-on: docker - if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 From a45becb2abe83f2f8361aa032366ef006d31e7d2 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Fri, 5 Jul 2024 23:58:18 +0200 Subject: [PATCH 18/30] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f67a607..dea61ce 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # Morffix -Morph and fix you Media distributedly with ffmpeg +Morph and fix your Media distributedly with ffmpeg From 265b2c16335ed0cc7efd83e31fe0a45615c783b4 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 01:20:01 +0200 Subject: [PATCH 19/30] Add System Files --- systemd/morffix-worker.service | 18 ++++++++++++++++++ systemd/morffix.service | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 systemd/morffix-worker.service create mode 100644 systemd/morffix.service diff --git a/systemd/morffix-worker.service b/systemd/morffix-worker.service new file mode 100644 index 0000000..be423a2 --- /dev/null +++ b/systemd/morffix-worker.service @@ -0,0 +1,18 @@ +[Unit] +Description=morffix worker +ConditionPathExists=/opt/morffix/morffix +After=network.target +Wants=morffix.service + +[Service] +Type=simple +User=root +LimitNOFILE=1024 +Restart=on-failure +RestartSec=10 +StartLimitIntervalSec=60 +WorkingDirectory=/opt/morffix +ExecStart=/opt/morffix/morffix + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemd/morffix.service b/systemd/morffix.service new file mode 100644 index 0000000..72c7d43 --- /dev/null +++ b/systemd/morffix.service @@ -0,0 +1,19 @@ +[Unit] +Description=morffix +ConditionPathExists=/opt/morffix/morffix +After=network.target +Wants=postgresql@15-main.service + +[Service] +Type=simple +User=root +Group=media_library +LimitNOFILE=1024 +Restart=on-failure +RestartSec=10 +StartLimitIntervalSec=60 +WorkingDirectory=/opt/morffix +ExecStart=/opt/morffix/morffix -server + +[Install] +WantedBy=multi-user.target \ No newline at end of file From e8f7521fa4953a916e055ab180955e9b97735f16 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 01:37:51 +0200 Subject: [PATCH 20/30] compile for amd64 --- .forgejo/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index e0035ad..2f63773 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -7,7 +7,7 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '1.22' - - run: go build + - run: GOOS=linux GOARCH=amd64 go build - run: mkdir out - run: cp morffix out/morffix - uses: actions/forgejo-release@eb0fcc44a150c0de82e6fdb36752dd56bf27d017 From b99a818040b8812e5df644bda356464b5e98ce36 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 03:05:58 +0200 Subject: [PATCH 21/30] Cleanup old temp files on startup --- task/healthcheck.go | 2 +- task/transcode.go | 4 ++-- worker/worker.go | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/task/healthcheck.go b/task/healthcheck.go index a8ab6a2..9039634 100644 --- a/task/healthcheck.go +++ b/task/healthcheck.go @@ -17,7 +17,7 @@ func RunHealthCheck(conf config.Config, t *types.Task, data types.HealthCheckDat l := log.GetTaskLogger(t) // TODO Figure out how to get correct file ending - path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("%v-%v.mkv", t.ID, t.FileID)) + path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("morffix-health-%v-%v.mkv", t.ID, t.FileID)) // Set ffmpeg input path if len(t.FfmpegCommand.InputFiles) == 0 { diff --git a/task/transcode.go b/task/transcode.go index 3a1e863..9d6a143 100644 --- a/task/transcode.go +++ b/task/transcode.go @@ -17,8 +17,8 @@ func RunTranscode(conf config.Config, t *types.Task, data types.TranscodeData) { l := log.GetTaskLogger(t) // TODO Figure out how to get correct file ending - src_path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("src-%v-%v.mkv", t.ID, t.FileID)) - dst_path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("dst-%v-%v.mkv", t.ID, t.FileID)) + src_path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("morffix-src-%v-%v.mkv", t.ID, t.FileID)) + dst_path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("morffix-dst-%v-%v.mkv", t.ID, t.FileID)) // Set ffmpeg input path if len(t.FfmpegCommand.InputFiles) == 0 { diff --git a/worker/worker.go b/worker/worker.go index 07dbad6..411c872 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -5,6 +5,7 @@ import ( "io" "log/slog" "net/http" + "path/filepath" "git.lastassault.de/speatzle/morffix/rpc" @@ -36,6 +37,20 @@ func Start(_conf config.Config) { return } + slog.InfoContext(ctx, "Cleaning tmp Files...") + files, err := filepath.Glob("/tmp/morffix-*") + if err != nil { + slog.Error("Get tmp Files", "err", err) + return + } + for _, f := range files { + slog.InfoContext(ctx, "Deleting File", "path", f) + if err := os.Remove(f); err != nil { + slog.Error("Deleting tmp File", "err", err, "path", f) + return + } + } + sigs := make(chan os.Signal, 1) signal.Notify(sigs, os.Interrupt) exit := false From c37c26908db844b2cbb3dc1b53edfb8fe377d5df Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 03:07:06 +0200 Subject: [PATCH 22/30] Prevent Starting Task Twice --- constants/error.go | 1 + task/task.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/constants/error.go b/constants/error.go index c375e01..341148c 100644 --- a/constants/error.go +++ b/constants/error.go @@ -3,3 +3,4 @@ package constants import "fmt" var ErrTaskDoesNotExist = fmt.Errorf("Task does not Exist") +var ErrTaskIsAlreadyRunning = fmt.Errorf("Task is Already Running") diff --git a/task/task.go b/task/task.go index 6e2ad61..f6c687c 100644 --- a/task/task.go +++ b/task/task.go @@ -17,6 +17,11 @@ func StartTask(conf config.Config, data types.TaskStart) error { taskMutex.Lock() defer taskMutex.Unlock() + _, ok := tasks[data.ID] + if ok { + return constants.ErrTaskIsAlreadyRunning + } + tasks[data.ID] = &types.Task{ ID: data.ID, Type: data.Type, From a9d901da719e508e5c303abce8d271c83c3e38d7 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 03:08:58 +0200 Subject: [PATCH 23/30] Also Count Unkown Tasks as Assigned --- server/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/task.go b/server/task.go index 8161ca1..4d430a1 100644 --- a/server/task.go +++ b/server/task.go @@ -358,7 +358,7 @@ func assignQueuedTasks(ctx context.Context) error { } if Workers[i].Connected { var count int - err := db.QueryRow(ctx, "SELECT COUNT(*) FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4)", i, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING).Scan(&count) + err := db.QueryRow(ctx, "SELECT COUNT(*) FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", i, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING).Scan(&count) if err != nil { return fmt.Errorf("Error Querying Worker Task Count: %w", err) } From 7574628a1d1c0b9e4617ddef9d9b548a07ddb5bb Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 03:09:34 +0200 Subject: [PATCH 24/30] check worker for unkown tasks --- server/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/worker.go b/server/worker.go index 573afea..2305cff 100644 --- a/server/worker.go +++ b/server/worker.go @@ -180,7 +180,7 @@ func updateWorkerTaskStatus(ctx context.Context) { if Workers[uuid].Connected { w := Workers[uuid] - rows, err := db.Query(ctx, "SELECT id, COALESCE(CARDINALITY(log),0) as log_offset FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4)", uuid, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING) + rows, err := db.Query(ctx, "SELECT id, COALESCE(CARDINALITY(log),0) as log_offset FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", uuid, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING) if err != nil { slog.ErrorContext(ctx, "Error Getting Tasks for Worker", "err", err, "worker_id", uuid) return From 78ab8c9daf18e9d5a962b8c718683dbe2828533f Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 15:54:49 +0200 Subject: [PATCH 25/30] Handle Unkown Tasks and RPC Error way better --- constants/error.go | 1 + rpc/call.go | 3 ++- server/task.go | 20 +++++++++++++++++++- server/worker.go | 13 +++++++++++-- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/constants/error.go b/constants/error.go index 341148c..bb3e278 100644 --- a/constants/error.go +++ b/constants/error.go @@ -4,3 +4,4 @@ import "fmt" var ErrTaskDoesNotExist = fmt.Errorf("Task does not Exist") var ErrTaskIsAlreadyRunning = fmt.Errorf("Task is Already Running") +var ErrRPCRequestTimeout = fmt.Errorf("Request timed out") diff --git a/rpc/call.go b/rpc/call.go index 71696de..037f384 100644 --- a/rpc/call.go +++ b/rpc/call.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "git.lastassault.de/speatzle/morffix/constants" "github.com/google/uuid" "nhooyr.io/websocket" ) @@ -89,7 +90,7 @@ func (s *server) Call(ctx context.Context, c *websocket.Conn, method string, par // remove request from map delete(s.requests, id) - return nil, fmt.Errorf("Request timed out") + return nil, constants.ErrRPCRequestTimeout } } diff --git a/server/task.go b/server/task.go index 4d430a1..d86d899 100644 --- a/server/task.go +++ b/server/task.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "log/slog" "net/http" "slices" "strconv" + "strings" "time" "git.lastassault.de/speatzle/morffix/constants" @@ -387,7 +389,23 @@ func assignQueuedTasks(ctx context.Context) error { } _, err = rpcServer.Call(ctx, Workers[i].Conn, "task-start", taskStart, nil) - if err != nil { + if strings.HasSuffix(err.Error(), constants.ErrTaskIsAlreadyRunning.Error()) { + // Task was started previously but something went wrong and we are out of sync + slog.WarnContext(ctx, "Task is apparently already Running on this Worker, thats bad", "task_id", taskStart.ID, "worker", Workers[i].Name) + + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_ASSIGNED, []string{fmt.Sprintf("%v MASTER: Task Start, Task Already Running!", time.Now())}) + if err != nil { + return fmt.Errorf("Updating Task status during already running error: %w", err) + } + } else if errors.Is(err, constants.ErrRPCRequestTimeout) { + // We really don't know whats going on, might be slow response, oom, disk full or a bug + slog.WarnContext(ctx, "Task start Timed Out", "task_id", taskStart.ID, "worker", Workers[i].Name) + + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskStart.ID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Task Start RPC Call Timed Out!", time.Now())}) + if err != nil { + return fmt.Errorf("Updating Unknown Task Status due to Timeout while starting Task: %w", err) + } + } else if err != nil { return fmt.Errorf("Error Starting Task: %w", err) } diff --git a/server/worker.go b/server/worker.go index 2305cff..b501165 100644 --- a/server/worker.go +++ b/server/worker.go @@ -2,6 +2,8 @@ package server import ( "context" + "errors" + "fmt" "log/slog" "net/http" "strings" @@ -207,7 +209,7 @@ func updateWorkerTaskStatus(ctx context.Context) { if strings.HasSuffix(err.Error(), constants.ErrTaskDoesNotExist.Error()) { // Worker says it does not know of this task, mark it failed so that we don't asks the worker about it again and again slog.ErrorContext(ctx, "Task is unknown by worker, Failing...", "err", err, "id", taskID) - _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskID, constants.TASK_STATUS_FAILED, []string{"Task Failed because it is unknown to Assigned Worker"}) + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskID, constants.TASK_STATUS_FAILED, []string{fmt.Sprintf("%v MASTER: Task Failed because it is unknown to Assigned Worker", time.Now())}) if err != nil { slog.ErrorContext(ctx, "Updating Failed Task Status", "err", err) return @@ -215,10 +217,17 @@ func updateWorkerTaskStatus(ctx context.Context) { slog.Info("Updating task done", "id", taskID, "status", constants.TASK_STATUS_FAILED) return + } else if errors.Is(err, constants.ErrRPCRequestTimeout) { + // We really don't know whats going on, might be slow response, oom, disk full or a bug + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Status RPC Call Timed Out!", time.Now())}) + if err != nil { + slog.ErrorContext(ctx, "Updating Unknown Task Status due to Timeout", "err", err) + return + } } else { slog.ErrorContext(ctx, "Getting Task Status", "err", err) - _, err = db.Exec(ctx, "UPDATE tasks SET status = $2 WHERE id = $1", taskID, constants.TASK_STATUS_UNKNOWN) + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskID, constants.TASK_STATUS_UNKNOWN, []string{fmt.Sprintf("%v MASTER: Status RPC Call Error: %v", time.Now(), err.Error())}) if err != nil { slog.ErrorContext(ctx, "Updating Unknown Task Status", "err", err) return From 24e11d9103a7c31bf6c70df84ad43dff55c2cd36 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 15:59:34 +0200 Subject: [PATCH 26/30] save log offeset so that the master can also log without dropping worker log lines --- migrations/000015_alter_tasks_table_log_offset.down.sql | 2 ++ migrations/000015_alter_tasks_table_log_offset.up.sql | 2 ++ server/worker.go | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 migrations/000015_alter_tasks_table_log_offset.down.sql create mode 100644 migrations/000015_alter_tasks_table_log_offset.up.sql diff --git a/migrations/000015_alter_tasks_table_log_offset.down.sql b/migrations/000015_alter_tasks_table_log_offset.down.sql new file mode 100644 index 0000000..bfbd448 --- /dev/null +++ b/migrations/000015_alter_tasks_table_log_offset.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE tasks +DROP IF EXISTS log_offset; diff --git a/migrations/000015_alter_tasks_table_log_offset.up.sql b/migrations/000015_alter_tasks_table_log_offset.up.sql new file mode 100644 index 0000000..7543ea9 --- /dev/null +++ b/migrations/000015_alter_tasks_table_log_offset.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE tasks +ADD log_offset integer NOT NULL DEFAULT COALESCE(CARDINALITY(log),0); diff --git a/server/worker.go b/server/worker.go index b501165..ed8322e 100644 --- a/server/worker.go +++ b/server/worker.go @@ -182,7 +182,7 @@ func updateWorkerTaskStatus(ctx context.Context) { if Workers[uuid].Connected { w := Workers[uuid] - rows, err := db.Query(ctx, "SELECT id, COALESCE(CARDINALITY(log),0) as log_offset FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", uuid, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING) + rows, err := db.Query(ctx, "SELECT id, log_offset FROM tasks WHERE worker_id = $1 AND (status = $2 OR status = $3 OR status = $4 OR status = $5)", uuid, constants.TASK_STATUS_UNKNOWN, constants.TASK_STATUS_ASSIGNED, constants.TASK_STATUS_RUNNING, constants.TASK_STATUS_WAITING) if err != nil { slog.ErrorContext(ctx, "Error Getting Tasks for Worker", "err", err, "worker_id", uuid) return @@ -236,7 +236,7 @@ func updateWorkerTaskStatus(ctx context.Context) { } } - _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3 WHERE id = $1", taskID, ts.Task.Status, ts.Task.Log) + _, err = db.Exec(ctx, "UPDATE tasks SET status = $2, log = log || $3, log_offset = log_offset + $4 WHERE id = $1", taskID, ts.Task.Status, ts.Task.Log, len(ts.Task.Log)) if err != nil { slog.ErrorContext(ctx, "Updating Task Status", "err", err) return From eb3cfbfdba5082cfd33b1cdcff7bbb5e14998e39 Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 16:14:28 +0200 Subject: [PATCH 27/30] fix migration --- migrations/000015_alter_tasks_table_log_offset.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/000015_alter_tasks_table_log_offset.up.sql b/migrations/000015_alter_tasks_table_log_offset.up.sql index 7543ea9..1e6a5d5 100644 --- a/migrations/000015_alter_tasks_table_log_offset.up.sql +++ b/migrations/000015_alter_tasks_table_log_offset.up.sql @@ -1,2 +1,2 @@ ALTER TABLE tasks -ADD log_offset integer NOT NULL DEFAULT COALESCE(CARDINALITY(log),0); +ADD log_offset integer NOT NULL DEFAULT 0; From 8046093e9ee6aedb80b48521d1a8209473ffd07f Mon Sep 17 00:00:00 2001 From: Samuel Lorch Date: Sat, 6 Jul 2024 16:35:59 +0200 Subject: [PATCH 28/30] Add Transcode Status Filter for Task Creation --- server/task.go | 3 ++- tmpl/tasks.tmpl | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/task.go b/server/task.go index d86d899..bfc6d16 100644 --- a/server/task.go +++ b/server/task.go @@ -260,6 +260,7 @@ func createTask(ctx context.Context, r *http.Request) error { } library := r.FormValue("library") health := r.FormValue("health") + transcode := r.FormValue("transcode") typ, err := strconv.Atoi(r.FormValue("type")) if err != nil { return fmt.Errorf("Parsing Task Type: %w", err) @@ -271,7 +272,7 @@ func createTask(ctx context.Context, r *http.Request) error { slog.Info("Got Task Create", "library", library, "health", health, "type", typ) - rows, err := db.Query(r.Context(), "SELECT id, path, size, status, health, transcode, md5, updated_at FROM files where library_id = $1 AND status = $2 AND (-1 = $3 OR health = $3)", library, constants.FILE_STATUS_EXISTS, health) + rows, err := db.Query(r.Context(), "SELECT id, path, size, status, health, transcode, md5, updated_at FROM files where library_id = $1 AND status = $2 AND (-1 = $3 OR health = $3) AND (-1 = $4 OR transcode = $4)", library, constants.FILE_STATUS_EXISTS, health, transcode) if err != nil { return fmt.Errorf("Query Files: %w", err) } diff --git a/tmpl/tasks.tmpl b/tmpl/tasks.tmpl index 647bca2..6220bcd 100644 --- a/tmpl/tasks.tmpl +++ b/tmpl/tasks.tmpl @@ -15,6 +15,14 @@ + +