From 68ccc2f4fc31d0d175de62deb58f30da0e393750 Mon Sep 17 00:00:00 2001 From: Twopic2 Date: Wed, 25 Feb 2026 00:27:36 -0800 Subject: [PATCH] first successful export --- .gitignore | 3 +- DockerFile | 24 +++++++++++++++ docker-compose.yaml | 24 +++++++++++++++ frontend/vite.config.ts | 10 +++---- go.mod | 7 ----- go.sum | 8 ----- handlers/analyze.go | 33 ++++----------------- handlers/download.go | 26 ++++++++--------- handlers/export.go | 59 +++++-------------------------------- handlers/upload.go | 22 ++++---------- main.go | 12 ++------ store/store.go | 62 ++++++++++++++++++++++++--------------- transcode/transcode.go | 65 +++++++++++++++++++++-------------------- ws/hub.go | 15 ++-------- 14 files changed, 163 insertions(+), 207 deletions(-) create mode 100644 DockerFile create mode 100644 docker-compose.yaml diff --git a/.gitignore b/.gitignore index 756e32f..da29371 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -/frontend/node_modules/ \ No newline at end of file +/frontend/node_modules/ +./aroll \ No newline at end of file diff --git a/DockerFile b/DockerFile new file mode 100644 index 0000000..9a82749 --- /dev/null +++ b/DockerFile @@ -0,0 +1,24 @@ +# Stage 1: Build frontend +FROM node:20-alpine AS frontend-builder +WORKDIR /app/frontend +COPY frontend/package.json frontend/package-lock.json* ./ +RUN npm install +COPY frontend/ . +RUN npm run build + +# Stage 2: Build Go binary +FROM golang:1.25.7-alpine AS go-builder +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o server . + +# Stage 3: Final image +FROM alpine:latest +RUN apk add --no-cache ffmpeg +WORKDIR /app +COPY --from=go-builder /app/server . +COPY --from=frontend-builder /app/frontend/dist ./frontend/dist +EXPOSE 8080 +CMD ["./server"] diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..6bd12bd --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,24 @@ +services: + frontend: + image: node:20-alpine + working_dir: /app + volumes: + - ./frontend:/app + - /app/node_modules + ports: + - "5173:5173" + command: sh -c "npm install && npm run dev -- --host" + depends_on: + - app + + app: + build: + context: . + dockerfile: DockerFile + ports: + - "8080:8080" + volumes: + - aroll_tmp:/tmp/aroll-uploads + +volumes: + aroll_tmp: diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 7d248fa..9288b65 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -5,11 +5,11 @@ export default defineConfig({ plugins: [react()], server: { proxy: { - '/upload': 'http://localhost:8080', - '/analyze': 'http://localhost:8080', - '/export': 'http://localhost:8080', - '/download': 'http://localhost:8080', - '/ws': { target: 'ws://localhost:8080', ws: true }, + '/upload': { target: 'http://app:8080', changeOrigin: true }, + '/analyze': { target: 'http://app:8080', changeOrigin: true }, + '/export': { target: 'http://app:8080', changeOrigin: true }, + '/download': { target: 'http://app:8080', changeOrigin: true }, + '/ws': { target: 'ws://app:8080', ws: true, changeOrigin: true }, }, }, }) diff --git a/go.mod b/go.mod index 7ea9b41..0ca6121 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,3 @@ module aroll go 1.25.7 require github.com/gorilla/websocket v1.5.3 - -require ( - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/redis/go-redis/v9 v9.18.0 // indirect - go.uber.org/atomic v1.11.0 // indirect -) diff --git a/go.sum b/go.sum index 049c9b9..25a9fc4 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,2 @@ -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= -github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= diff --git a/handlers/analyze.go b/handlers/analyze.go index 59f4e22..c8abc25 100644 --- a/handlers/analyze.go +++ b/handlers/analyze.go @@ -1,11 +1,8 @@ package handlers import ( - "context" "encoding/json" "net/http" - "os" - "path/filepath" "aroll/store" "aroll/transcode" @@ -19,8 +16,6 @@ type analyzeRequest struct { Padding float64 `json:"padding"` } -// AnalyzeHandler fetches the video bytes from Redis, writes them to a short-lived -// temp file for FFmpeg, runs silence detection, then deletes the temp file. func AnalyzeHandler(st *store.Store, hub *ws.Hub) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -34,33 +29,15 @@ func AnalyzeHandler(st *store.Store, hub *ws.Hub) http.HandlerFunc { return } - data, err := st.Get(context.Background(), req.Filename) - if err != nil { - http.Error(w, "file not found in store (may have expired)", http.StatusNotFound) + path, ok := st.Path(req.Filename) + if !ok { + http.Error(w, "file not found", http.StatusNotFound) return } - ext := filepath.Ext(req.Filename) - go func() { - // Write to a temp file — FFmpeg needs a file path, not a byte slice. - // This file exists only for the duration of the FFmpeg scan. - tmp, err := os.CreateTemp("", "aroll-analyze-*"+ext) - if err != nil { - broadcastError(hub, "create temp: "+err.Error()) - return - } - defer os.Remove(tmp.Name()) - - if _, err := tmp.Write(data); err != nil { - tmp.Close() - broadcastError(hub, "write temp: "+err.Error()) - return - } - tmp.Close() - - _, err = transcode.DetectSpeechSegments( - tmp.Name(), + _, err := transcode.DetectSpeechSegments( + path, req.NoiseDb, req.MinSilence, req.Padding, diff --git a/handlers/download.go b/handlers/download.go index cf34edf..fdf9f38 100644 --- a/handlers/download.go +++ b/handlers/download.go @@ -1,40 +1,38 @@ package handlers import ( - "context" "net/http" - "strconv" + "os" "strings" "aroll/store" ) -// DownloadHandler fetches the exported video from Redis, streams it to the -// browser, then immediately deletes the key. One download per export. func DownloadHandler(st *store.Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { key := strings.TrimPrefix(r.URL.Path, "/") - // Only serve keys we created in ExportHandler if !strings.HasPrefix(key, "output-") { http.NotFound(w, r) return } - ctx := context.Background() - - data, err := st.Get(ctx, key) - if err != nil { + path, ok := st.Path(key) + if !ok { http.Error(w, "file not found or already downloaded", http.StatusNotFound) return } + defer st.Delete(key) - // Delete from Redis now — one download only - defer st.Delete(ctx, key) + f, err := os.Open(path) + if err != nil { + http.Error(w, "file not found", http.StatusNotFound) + return + } + defer f.Close() - w.Header().Set("Content-Type", "video/mp4") + fi, _ := f.Stat() w.Header().Set("Content-Disposition", `attachment; filename="aroll-cut.mp4"`) - w.Header().Set("Content-Length", strconv.Itoa(len(data))) - w.Write(data) + http.ServeContent(w, r, "aroll-cut.mp4", fi.ModTime(), f) } } diff --git a/handlers/export.go b/handlers/export.go index 59886b9..a91e51c 100644 --- a/handlers/export.go +++ b/handlers/export.go @@ -1,11 +1,9 @@ package handlers import ( - "context" "encoding/json" "net/http" "os" - "path/filepath" "aroll/store" "aroll/transcode" @@ -17,9 +15,6 @@ type exportRequest struct { Segments []transcode.Segment `json:"segments"` } -// ExportHandler fetches the input video from Redis, runs FFmpeg to cut the -// silence, stores the output back in Redis, then cleans up the temp files. -// The output key is sent to the frontend via a "done" WebSocket message. func ExportHandler(st *store.Store, hub *ws.Hub) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -37,61 +32,23 @@ func ExportHandler(st *store.Store, hub *ws.Hub) http.HandlerFunc { return } - data, err := st.Get(context.Background(), req.Filename) - if err != nil { - http.Error(w, "file not found in store (may have expired)", http.StatusNotFound) + inPath, ok := st.Path(req.Filename) + if !ok { + http.Error(w, "file not found", http.StatusNotFound) return } - ext := filepath.Ext(req.Filename) - go func() { - // Write input bytes to a temp file for FFmpeg - inTmp, err := os.CreateTemp("", "aroll-in-*"+ext) - if err != nil { - broadcastError(hub, "create input temp: "+err.Error()) - return - } - defer os.Remove(inTmp.Name()) + /* ToDo: Make sure that other video formats gets transcoded */ + outputKey := "output-" + store.NewID() + ".mp4" + outPath := st.Allocate(outputKey) - if _, err := inTmp.Write(data); err != nil { - inTmp.Close() - broadcastError(hub, "write input temp: "+err.Error()) - return - } - inTmp.Close() - - // Create an output temp file for FFmpeg to write into - outTmp, err := os.CreateTemp("", "aroll-out-*.mp4") - if err != nil { - broadcastError(hub, "create output temp: "+err.Error()) - return - } - outTmp.Close() - defer os.Remove(outTmp.Name()) - - // Run FFmpeg — progress streamed via WebSocket - if err := transcode.ExportSegments( - inTmp.Name(), outTmp.Name(), req.Segments, hub.Broadcast, - ); err != nil { + if err := transcode.ExportSegments(inPath, outPath, req.Segments, hub.Broadcast); err != nil { + os.Remove(outPath) broadcastError(hub, err.Error()) return } - // Read the output bytes and store them in Redis - outData, err := os.ReadFile(outTmp.Name()) - if err != nil { - broadcastError(hub, "read output: "+err.Error()) - return - } - - outputKey := "output-" + store.NewID() - if err := st.Set(context.Background(), outputKey, outData); err != nil { - broadcastError(hub, "store output: "+err.Error()) - return - } - - // Tell the frontend where to download from msg, _ := json.Marshal(map[string]string{ "type": "done", "message": outputKey, diff --git a/handlers/upload.go b/handlers/upload.go index 91fa7af..06a0f9d 100644 --- a/handlers/upload.go +++ b/handlers/upload.go @@ -1,18 +1,13 @@ package handlers import ( - "context" "encoding/json" - "io" "net/http" "path/filepath" "aroll/store" ) -// UploadHandler reads the video into memory and stores it in Redis under a -// random key. The key is returned to the frontend as "filename". -// Nothing is written to disk — the file lives only in Redis with a TTL. func UploadHandler(st *store.Store) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -21,10 +16,11 @@ func UploadHandler(st *store.Store) http.HandlerFunc { } r.Body = http.MaxBytesReader(w, r.Body, 4<<30) - if err := r.ParseMultipartForm(64 << 20); err != nil { + if err := r.ParseMultipartForm(32 << 20); err != nil { http.Error(w, "parse form: "+err.Error(), http.StatusBadRequest) return } + defer r.MultipartForm.RemoveAll() file, header, err := r.FormFile("video") if err != nil { @@ -33,22 +29,14 @@ func UploadHandler(st *store.Store) http.HandlerFunc { } defer file.Close() - data, err := io.ReadAll(file) - if err != nil { - http.Error(w, "read file: "+err.Error(), http.StatusInternalServerError) - return - } - ext := filepath.Ext(header.Filename) if ext == "" { ext = ".mp4" } - // Key format: "video-" e.g. "video-a3f9...2b.mp4" - key := "video-" + store.NewID() + ext - - if err := st.Set(context.Background(), key, data); err != nil { - http.Error(w, "store: "+err.Error(), http.StatusInternalServerError) + key, err := st.Save(file, ext) + if err != nil { + http.Error(w, "save: "+err.Error(), http.StatusInternalServerError) return } diff --git a/main.go b/main.go index 39bb802..38e8318 100644 --- a/main.go +++ b/main.go @@ -11,16 +11,11 @@ import ( ) func main() { - redisAddr := "localhost:6379" - if addr := os.Getenv("REDIS_ADDR"); addr != "" { - redisAddr = addr - } - - st, err := store.New(redisAddr) + st, err := store.New() if err != nil { - log.Fatalf("redis: %v", err) + log.Fatalf("store: %v", err) } - log.Printf("Connected to Redis at %s", redisAddr) + log.Println("Temp storage ready") hub := ws.NewHub() go hub.Run() @@ -33,7 +28,6 @@ func main() { mux.HandleFunc("/export", handlers.ExportHandler(st, hub)) mux.Handle("/download/", http.StripPrefix("/download/", handlers.DownloadHandler(st))) - // Serve the React build in production if _, err := os.Stat("frontend/dist"); err == nil { mux.Handle("/", http.FileServer(http.Dir("frontend/dist"))) } diff --git a/store/store.go b/store/store.go index 43aecba..16d75a0 100644 --- a/store/store.go +++ b/store/store.go @@ -1,46 +1,62 @@ package store import ( - "context" "crypto/rand" "encoding/hex" "fmt" - "time" - - "github.com/redis/go-redis/v9" + "io" + "os" + "path/filepath" ) -// TTL is how long any stored file lives in Redis before automatic deletion. -const TTL = 2 * time.Hour - type Store struct { - rdb *redis.Client + dir string } -func New(addr string) (*Store, error) { - rdb := redis.NewClient(&redis.Options{Addr: addr}) - if err := rdb.Ping(context.Background()).Err(); err != nil { - return nil, fmt.Errorf("redis connect %s: %w", addr, err) +func New() (*Store, error) { + dir := filepath.Join(os.TempDir(), "aroll-uploads") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("create upload dir: %w", err) } - return &Store{rdb: rdb}, nil + return &Store{dir: dir}, nil } -// Set stores binary data under key with the global TTL. -func (s *Store) Set(ctx context.Context, key string, data []byte) error { - return s.rdb.Set(ctx, key, data, TTL).Err() +// Save streams r into a new file in the store and returns its key. +func (s *Store) Save(r io.Reader, ext string) (string, error) { + key := "video-" + NewID() + ext + path := filepath.Join(s.dir, key) + f, err := os.Create(path) + if err != nil { + return "", fmt.Errorf("create file: %w", err) + } + defer f.Close() + if _, err := io.Copy(f, r); err != nil { + os.Remove(path) + return "", fmt.Errorf("write file: %w", err) + } + return key, nil } -// Get retrieves binary data by key. -func (s *Store) Get(ctx context.Context, key string) ([]byte, error) { - return s.rdb.Get(ctx, key).Bytes() +// Path returns the absolute path for a key, if the file exists. +func (s *Store) Path(key string) (string, bool) { + path := filepath.Join(s.dir, key) + if _, err := os.Stat(path); err != nil { + return "", false + } + return path, true } -// Delete removes a key immediately (called after download). -func (s *Store) Delete(ctx context.Context, key string) { - s.rdb.Del(ctx, key) +// Allocate returns a path for a new key without creating the file. +// Used by ExportHandler so ffmpeg can write directly to the store. +func (s *Store) Allocate(key string) string { + return filepath.Join(s.dir, key) +} + +// Delete removes the file for a key. +func (s *Store) Delete(key string) { + os.Remove(filepath.Join(s.dir, key)) } -// NewID generates a random hex ID for use as a Redis key suffix. func NewID() string { b := make([]byte, 16) rand.Read(b) diff --git a/transcode/transcode.go b/transcode/transcode.go index 2ecdcb0..4e8b10e 100644 --- a/transcode/transcode.go +++ b/transcode/transcode.go @@ -8,6 +8,7 @@ import ( "os/exec" "regexp" "strconv" + "strings" ) // Segment is a time interval [Start, End) in seconds. @@ -24,8 +25,8 @@ type wsMsg struct { Message string `json:"message,omitempty"` } -func send(msg wsMsg, broadcast func([]byte)) { - data, _ := json.Marshal(msg) +func send(msg *wsMsg, broadcast func([]byte)) { + data, _ := json.Marshal(&msg) broadcast(data) } @@ -36,11 +37,7 @@ func send(msg wsMsg, broadcast func([]byte)) { // DetectSpeechSegments runs ffmpeg's silencedetect filter and returns the // non-silent (speech) segments. A "segments" WebSocket message is broadcast // when detection finishes. -func DetectSpeechSegments( - inputPath string, - noiseDb, minDuration, padding float64, - broadcast func([]byte), -) ([]Segment, error) { +func DetectSpeechSegments(inputPath string, noiseDb, minDuration, padding float64, broadcast func([]byte)) ([]Segment, error) { filter := fmt.Sprintf("silencedetect=noise=%.0fdB:d=%.2f", noiseDb, minDuration) cmd := exec.Command("ffmpeg", @@ -91,9 +88,9 @@ func DetectSpeechSegments( return nil, fmt.Errorf("ffmpeg: %w", err) } - segments := invertSilences(silences, totalDuration, padding) + segments := removeSilence(silences, totalDuration, padding) - send(wsMsg{ + send(&wsMsg{ Type: "segments", Segments: segments, Duration: totalDuration, @@ -104,9 +101,9 @@ func DetectSpeechSegments( type silenceInterval struct{ start, end float64 } -// invertSilences turns silence regions into speech regions, with a small +// removeSilence turns silence regions into speech regions, with a small // padding buffer so words at the edges don't get clipped. -func invertSilences(silences []silenceInterval, totalDuration, padding float64) []Segment { +func removeSilence(silences []silenceInterval, totalDuration, padding float64) []Segment { if len(silences) == 0 { return []Segment{{Start: 0, End: totalDuration}} } @@ -136,37 +133,41 @@ func invertSilences(silences []silenceInterval, totalDuration, padding float64) return segments } -// ────────────────────────────────────────────────────────────────────────────── -// Export -// ────────────────────────────────────────────────────────────────────────────── - -// ExportSegments concatenates the given segments using ffmpeg's concat demuxer -// (stream-copy, no re-encode). Progress is streamed via broadcast. func ExportSegments( inputPath, outputPath string, segments []Segment, broadcast func([]byte), ) error { - concatPath := outputPath + ".concat.txt" - f, err := os.Create(concatPath) - if err != nil { - return fmt.Errorf("create concat file: %w", err) - } - defer os.Remove(concatPath) - totalDuration := 0.0 for _, seg := range segments { totalDuration += seg.End - seg.Start - fmt.Fprintf(f, "file '%s'\ninpoint %.6f\noutpoint %.6f\n", - inputPath, seg.Start, seg.End) } - f.Close() + + // Build a filter_complex that trims each segment and resets timestamps, + // then concatenates them. This avoids non-monotonic DTS issues that occur + // when stream-copying segments with their original timestamps. + var filterParts []string + var concatInputs string + for i, seg := range segments { + filterParts = append(filterParts, + fmt.Sprintf("[0:v]trim=start=%.6f:end=%.6f,setpts=PTS-STARTPTS[v%d]", seg.Start, seg.End, i), + fmt.Sprintf("[0:a]atrim=start=%.6f:end=%.6f,asetpts=PTS-STARTPTS[a%d]", seg.Start, seg.End, i), + ) + concatInputs += fmt.Sprintf("[v%d][a%d]", i, i) + } + filterParts = append(filterParts, + fmt.Sprintf("%sconcat=n=%d:v=1:a=1[outv][outa]", concatInputs, len(segments)), + ) + filterComplex := strings.Join(filterParts, ";") cmd := exec.Command("ffmpeg", - "-f", "concat", - "-safe", "0", - "-i", concatPath, - "-c", "copy", + "-i", inputPath, + "-filter_complex", filterComplex, + "-map", "[outv]", + "-map", "[outa]", + // around 80% compression + "-c:v", "libx264", "-crf", "28", "-preset", "fast", + "-c:a", "aac", "-b:a", "128k", "-progress", "pipe:1", "-y", outputPath, @@ -194,7 +195,7 @@ func ExportSegments( if pct > 100 { pct = 100 } - send(wsMsg{Type: "progress", Percent: pct}, broadcast) + send(&wsMsg{Type: "progress", Percent: pct}, broadcast) } } diff --git a/ws/hub.go b/ws/hub.go index d6ddea9..ba813d3 100644 --- a/ws/hub.go +++ b/ws/hub.go @@ -1,7 +1,5 @@ package ws -import "sync" - // Client represents a single WebSocket connection. // The handler owns the actual *websocket.Conn; the hub only needs the send channel. type Client struct { @@ -10,7 +8,6 @@ type Client struct { // Hub manages all active clients and routes broadcast messages to them. type Hub struct { - mu sync.Mutex clients map[*Client]struct{} Register chan *Client Unregister chan *Client @@ -19,10 +16,10 @@ type Hub struct { func NewHub() *Hub { return &Hub{ - clients: make(map[*Client]struct{}), - Register: make(chan *Client, 8), + clients: make(map[*Client]struct{}), + Register: make(chan *Client, 8), Unregister: make(chan *Client, 8), - broadcast: make(chan []byte, 512), + broadcast: make(chan []byte, 512), } } @@ -30,20 +27,15 @@ func (h *Hub) Run() { for { select { case c := <-h.Register: - h.mu.Lock() h.clients[c] = struct{}{} - h.mu.Unlock() case c := <-h.Unregister: - h.mu.Lock() if _, ok := h.clients[c]; ok { delete(h.clients, c) close(c.Send) } - h.mu.Unlock() case msg := <-h.broadcast: - h.mu.Lock() for c := range h.clients { select { case c.Send <- msg: @@ -52,7 +44,6 @@ func (h *Hub) Run() { close(c.Send) } } - h.mu.Unlock() } } }