package task import ( "bufio" "bytes" "context" "crypto/md5" "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "slices" "sync" "time" "git.lastassault.de/speatzle/morffix/config" "git.lastassault.de/speatzle/morffix/constants" "git.lastassault.de/speatzle/morffix/task/log" "git.lastassault.de/speatzle/morffix/types" ) type countReader struct { io.Reader n int } func (w *countReader) Read(p []byte) (int, error) { n, err := w.Reader.Read(p) w.n += n return n, err } // dropCR drops a terminal \r from the data. func dropCR(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\r' { return data[0 : len(data)-1] } return data } func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } if i := bytes.IndexByte(data, '\n'); i >= 0 { // We have a full newline-terminated line. return i + 1, dropCR(data[0:i]), nil } if i := bytes.IndexByte(data, '\r'); i >= 0 { // We have a return line. return i + 1, dropCR(data[0:i]), nil } // If we're at EOF, we have a final, non-terminated line. Return it. if atEOF { return len(data), dropCR(data), nil } // Request more data. return 0, nil, nil } func RunHealthCheck(conf config.Config, t *types.Task, data types.HealthCheckData) { ctx := context.TODO() l := log.GetTaskLogger(t) // TODO Figure out how to get correct file ending path := filepath.Join(conf.Worker.TempDir, fmt.Sprintf("%v-%v.mkv", t.ID, t.FileID)) // Set ffmpeg input path if len(data.Command.InputFiles) == 0 { l.ErrorContext(ctx, "FFmpeg Command has no input files") return } data.Command.InputFiles[0].Path = path // TODO cleanup file when done defer func() { err := os.Remove(path) if err != nil { l.ErrorContext(ctx, "Removing File", "err", err, "path", path) } else { l.InfoContext(ctx, "File Removed Succesfully", "path", path) } }() l.InfoContext(ctx, "Starting File Download", "task_id", t.ID, "file_id", t.FileID, "path", path, "md5", t.FileMD5) err := func() error { out, err := os.Create(path) if err != nil { return fmt.Errorf("Creating File: %w", err) } defer out.Close() req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%v/files/%v", conf.Worker.Address, t.FileID), nil) if err != nil { return fmt.Errorf("New Request: %w", err) } req.Close = true var client = &http.Client{ Transport: &http.Transport{}, } resp, err := client.Do(req) if err != nil { return fmt.Errorf("Getting File: %w", err) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("Got HTTP Status Code: %v", resp.StatusCode) } req.Close = true // TODO Log at interval logs read // Calculate hash and write file at the same time hash := md5.New() tr := io.TeeReader(resp.Body, hash) cr := &countReader{Reader: tr} stopProgress := make(chan bool, 1) go func() { tik := time.NewTicker(time.Second) lastCount := 0 for { select { case <-tik.C: speed := cr.n - lastCount l.InfoContext(ctx, "Download Progress", "bytes", cr.n, "lastCount", lastCount, "length", resp.ContentLength, "speed", speed) lastCount = cr.n case <-stopProgress: tik.Stop() } } }() defer func() { stopProgress <- true }() defer resp.Body.Close() n, err := io.Copy(out, cr) if err != nil { return fmt.Errorf("Reading File: %w", err) } md5 := hash.Sum(nil) l.InfoContext(ctx, "Downloaded File", "bytes", n, "md5", md5) if slices.Compare[[]byte](md5, t.FileMD5) != 0 { return fmt.Errorf("Downloaded File does not match md5") } return nil }() if err != nil { l.ErrorContext(ctx, "File Download Failed", "err", err) return } l.InfoContext(ctx, "Running ffmpeg", "args", data.Command.GetArgs()) cmd := exec.CommandContext(ctx, conf.Worker.FFmpegPath, data.Command.GetArgs()...) var wg sync.WaitGroup stdout, err := cmd.StdoutPipe() if err != nil { l.ErrorContext(ctx, "Error getting StdoutPipe", "err", err) return } wg.Add(1) outScanner := bufio.NewScanner(stdout) outScanner.Split(scanLines) go func() { for outScanner.Scan() { l.InfoContext(ctx, outScanner.Text()) } wg.Done() }() stderr, err := cmd.StderrPipe() if err != nil { l.ErrorContext(ctx, "Error getting StderrPipe", "err", err) return } wg.Add(1) errScanner := bufio.NewScanner(stderr) errScanner.Split(scanLines) go func() { for errScanner.Scan() { l.InfoContext(ctx, errScanner.Text()) } wg.Done() }() err = cmd.Start() if err != nil { l.ErrorContext(ctx, "Error Starting ffmpeg", "err", err) return } wg.Wait() err = cmd.Wait() if err != nil { l.ErrorContext(ctx, "Error Running ffmpeg", "err", err) return } l.InfoContext(ctx, "Task Success") t.Status = constants.TASK_STATUS_SUCCESS }