Stream post videos to Spaces (#23)

Pass the spooled video file through instead of buffering it in RAM, and sign Spaces uploads as UNSIGNED-PAYLOAD so the client does not hash the body first.

Reviewed-on: #23
Co-authored-by: codegirl-007 <s.raide@gmail.com>

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.43.7
 	github.com/aws/aws-sdk-go-v2/credentials v1.19.37
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3
+	github.com/aws/smithy-go v1.27.8
 	github.com/bwmarrin/discordgo v0.29.0
 	github.com/go-chi/chi/v5 v5.3.1
 	github.com/google/uuid v1.6.0
@@ -27,7 +28,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect
-	github.com/aws/smithy-go v1.27.8 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect

internal/blob/spaces.go

@@ -8,9 +8,11 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go/middleware"
 )
 
 // Uploader stores public avatar objects.
@@ -81,10 +83,23 @@ func NewSpaces(cfg SpacesConfig) Uploader {
 		Region:                     cfg.Region,
 		Credentials:                credentials.NewStaticCredentialsProvider(cfg.Key, cfg.Secret, ""),
 		BaseEndpoint:               aws.String(cfg.Endpoint),
+		RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+		APIOptions: []func(*middleware.Stack) error{
+			spacesUnsignedPayload,
+		},
 	})
 	return &spaces{client: client, cfg: cfg}
 }
 
+// spacesUnsignedPayload signs Spaces PUTs as UNSIGNED-PAYLOAD so the client
+// can stream the body without hashing it first.
+func spacesUnsignedPayload(stack *middleware.Stack) error {
+	if err := v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware(stack); err != nil {
+		return v4.AddUnsignedPayloadMiddleware(stack)
+	}
+	return nil
+}
+
 func (s *spaces) Enabled() bool { return true }
 
 func (s *spaces) Upload(ctx context.Context, obj FileUpload) (string, error) {

internal/web/events.go

@@ -55,6 +55,9 @@ func (s *Server) postEvent(post, root *store.Post, author *store.User) events.Po
 	if n := len(post.Images); n > 0 {
 		ev.Images = make([]events.Image, 0, n)
 		for _, img := range post.Images {
+			if img.Kind == store.MediaKindVideo {
+				continue
+			}
 			ev.Images = append(ev.Images, events.Image{
 				URL:         img.PublicURL,
 				Description: img.Description,

internal/web/events_test.go

@@ -236,3 +236,21 @@ func assertPostEvent(t *testing.T, got, want events.PostEvent) {
 		t.Fatalf("event = %+v, want %+v", got, want)
 	}
 }
+
+func TestPostEventOmitsVideos(t *testing.T) {
+	t.Parallel()
+
+	srv, _ := newTestServer(t, Config{})
+	got := srv.postEvent(&store.Post{
+		ID:     "root-1",
+		Title:  "Clip",
+		Body:   "Photo and video.",
+		Images: []store.PostImage{
+			{PublicURL: "https://cdn.example/a.jpg", Description: "Still", Kind: store.MediaKindImage},
+			{PublicURL: "https://cdn.example/a.mp4", Description: "Walkthrough", Kind: store.MediaKindVideo},
+		},
+	}, nil, nil)
+	if len(got.Images) != 1 || got.Images[0].URL != "https://cdn.example/a.jpg" {
+		t.Fatalf("event images = %+v", got.Images)
+	}
+}

internal/web/post_images.go

@@ -28,7 +28,8 @@ import (
 const (
 	defaultRequestBodyBytes  = 3 << 20
 	postImageMaxFileBytes    = 5 << 20
-	postImageMaxRequestBytes = 22 << 20
+	postVideoMaxFileBytes    = 25 << 20
+	postImageMaxRequestBytes = 50 << 20 // 4 images + 1 video + form fields
 	postImageMultipartMemory = 2 << 20
 	postImageMaxSourceDim    = 6000
 	postImageMaxSourcePixels = 16_000_000
@@ -67,7 +68,7 @@ func parsePostMutationForm(w http.ResponseWriter, r *http.Request) (func(), bool
 	contentType := r.Header.Get("Content-Type")
 	mediaType, _, err := mime.ParseMediaType(contentType)
 	if err != nil && strings.HasPrefix(strings.ToLower(contentType), "multipart/") {
-		http.Error(w, "Could not read image upload.", http.StatusBadRequest)
+		http.Error(w, "Could not read upload.", http.StatusBadRequest)
 		return func() {}, false
 	}
 	if mediaType != "multipart/form-data" {
@@ -97,10 +98,10 @@ func writePostImageRequestError(w http.ResponseWriter, err error) {
 	}
 	var maxErr *http.MaxBytesError
 	if errors.As(err, &maxErr) {
-		http.Error(w, "Image upload is too large.", http.StatusRequestEntityTooLarge)
+		http.Error(w, "Upload is too large.", http.StatusRequestEntityTooLarge)
 		return
 	}
-	http.Error(w, "Could not read image upload.", http.StatusBadRequest)
+	http.Error(w, "Could not read upload.", http.StatusBadRequest)
 }
 
 func (s *Server) postImagesFromForm(
@@ -121,9 +122,23 @@ func (s *Server) postImagesFromForm(
 	if len(descriptions) > len(files) {
 		return nil, nil, invalidPostImage("Image descriptions do not match selected images.", nil)
 	}
-	if len(retained)+len(files) > store.MaxPostImages {
+	kinds := make([]string, len(files))
+	newImages, newVideos := 0, 0
+	for i, header := range files {
+		kinds[i] = sniffPostMedia(header)
+		if kinds[i] == store.MediaKindVideo {
+			newVideos++
+		} else {
+			newImages++
+		}
+	}
+	retainedImages, retainedVideos := countPostMedia(retained)
+	if retainedImages+newImages > store.MaxPostImages {
 		return nil, nil, invalidPostImage("You can attach up to 4 images.", nil)
 	}
+	if retainedVideos+newVideos > store.MaxPostVideos {
+		return nil, nil, invalidPostImage("You can attach one video.", nil)
+	}
 	if len(files) > 0 && !s.cfg.Blob.Enabled() {
 		return nil, nil, &postImageRequestError{
 			status:  http.StatusServiceUnavailable,
@@ -142,11 +157,68 @@ func (s *Server) postImagesFromForm(
 			s.deletePostImageObjects(newKeys)
 			return nil, nil, invalidPostImage("Image descriptions must be 500 characters or fewer.", nil)
 		}
-		prepared, err := preparePostImage(header)
+		item, objectKey, err := s.uploadPostMedia(ctx, postID, header, kinds[i], description)
 		if err != nil {
 			s.deletePostImageObjects(newKeys)
 			return nil, nil, err
 		}
+		newKeys = append(newKeys, objectKey)
+		images = append(images, item)
+	}
+	return images, newKeys, nil
+}
+
+func countPostMedia(items []store.PostImage) (images, videos int) {
+	for _, item := range items {
+		if item.Kind == store.MediaKindVideo {
+			videos++
+		} else {
+			images++
+		}
+	}
+	return images, videos
+}
+
+func (s *Server) uploadPostMedia(
+	ctx context.Context,
+	postID string,
+	header *multipart.FileHeader,
+	kind, description string,
+) (store.PostImage, string, error) {
+	if kind == store.MediaKindVideo {
+		prepared, err := preparePostVideo(header)
+		if err != nil {
+			return store.PostImage{}, "", err
+		}
+		defer prepared.body.Close()
+		mediaID := uuid.NewString()
+		objectKey := path.Join("post-videos", postID, mediaID+prepared.extension)
+		publicURL, err := s.cfg.Blob.Upload(ctx, blob.FileUpload{
+			Key:         objectKey,
+			Body:        prepared.body,
+			ContentType: prepared.contentType,
+			Size:        prepared.size,
+		})
+		if err != nil {
+			return store.PostImage{}, "", &postImageRequestError{
+				status:  http.StatusServiceUnavailable,
+				message: "Could not upload video. Try again later.",
+				cause:   err,
+			}
+		}
+		return store.PostImage{
+			ID:          mediaID,
+			PostID:      postID,
+			ObjectKey:   objectKey,
+			PublicURL:   publicURL,
+			Description: description,
+			Kind:        store.MediaKindVideo,
+		}, objectKey, nil
+	}
+	prepared, err := preparePostImage(header)
+	if err != nil {
+		return store.PostImage{}, "", err
+	}
 	imageID := uuid.NewString()
 	objectKey := path.Join("post-images", postID, imageID+prepared.extension)
 	publicURL, err := s.cfg.Blob.Upload(ctx, blob.FileUpload{
@@ -156,25 +228,22 @@ func (s *Server) postImagesFromForm(
 		Size:        int64(len(prepared.body)),
 	})
 	if err != nil {
-			s.deletePostImageObjects(newKeys)
-			return nil, nil, &postImageRequestError{
+		return store.PostImage{}, "", &postImageRequestError{
 			status:  http.StatusServiceUnavailable,
 			message: "Could not upload image. Try again later.",
 			cause:   err,
 		}
 	}
-		newKeys = append(newKeys, objectKey)
-		images = append(images, store.PostImage{
+	return store.PostImage{
 		ID:          imageID,
 		PostID:      postID,
 		ObjectKey:   objectKey,
 		PublicURL:   publicURL,
 		Description: description,
+		Kind:        store.MediaKindImage,
 		Width:       prepared.width,
 		Height:      prepared.height,
-		})
-	}
-	return images, newKeys, nil
+	}, objectKey, nil
 }
 
 func retainedPostImages(form *multipart.Form, existing []store.PostImage) ([]store.PostImage, error) {
@@ -298,6 +367,99 @@ func preparePostImage(header *multipart.FileHeader) (preparedPostImage, error) {
 	return result, nil
 }
 
+func sniffPostMedia(header *multipart.FileHeader) string {
+	if header == nil {
+		return store.MediaKindImage
+	}
+	file, err := header.Open()
+	if err != nil {
+		return store.MediaKindImage
+	}
+	defer file.Close()
+	peek := make([]byte, 512)
+	n, err := io.ReadFull(file, peek)
+	if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) {
+		return store.MediaKindImage
+	}
+	return mediaKindFromBytes(peek[:n])
+}
+
+func mediaKindFromBytes(raw []byte) string {
+	switch http.DetectContentType(raw) {
+	case "video/mp4", "video/webm":
+		return store.MediaKindVideo
+	case "image/jpeg", "image/png", "image/webp":
+		return store.MediaKindImage
+	}
+	if isMP4(raw) || isWebM(raw) {
+		return store.MediaKindVideo
+	}
+	return store.MediaKindImage
+}
+
+func isMP4(raw []byte) bool {
+	return len(raw) >= 8 && string(raw[4:8]) == "ftyp"
+}
+
+func isWebM(raw []byte) bool {
+	return len(raw) >= 4 && raw[0] == 0x1a && raw[1] == 0x45 && raw[2] == 0xdf && raw[3] == 0xa3
+}
+
+type preparedPostVideo struct {
+	body        io.ReadCloser
+	size        int64
+	extension   string
+	contentType string
+}
+
+func preparePostVideo(header *multipart.FileHeader) (preparedPostVideo, error) {
+	if header == nil {
+		return preparedPostVideo{}, invalidPostImage("Select a valid video.", nil)
+	}
+	if header.Size == 0 {
+		return preparedPostVideo{}, invalidPostImage("Videos cannot be empty.", nil)
+	}
+	if header.Size > postVideoMaxFileBytes {
+		return preparedPostVideo{}, &postImageRequestError{
+			status:  http.StatusRequestEntityTooLarge,
+			message: "Each video must be 25 MB or smaller.",
+		}
+	}
+	file, err := header.Open()
+	if err != nil {
+		return preparedPostVideo{}, invalidPostImage("Could not read video.", err)
+	}
+	peek := make([]byte, 512)
+	n, err := io.ReadFull(file, peek)
+	if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) {
+		file.Close()
+		return preparedPostVideo{}, invalidPostImage("Could not read video.", err)
+	}
+	if n == 0 {
+		file.Close()
+		return preparedPostVideo{}, invalidPostImage("Videos cannot be empty.", nil)
+	}
+	switch mediaKindFromBytes(peek[:n]) {
+	case store.MediaKindVideo:
+	default:
+		file.Close()
+		return preparedPostVideo{}, invalidPostImage("Videos must be MP4 or WebM.", nil)
+	}
+	if _, err := file.Seek(0, io.SeekStart); err != nil {
+		file.Close()
+		return preparedPostVideo{}, invalidPostImage("Could not read video.", err)
+	}
+	result := preparedPostVideo{body: file, size: header.Size}
+	if isWebM(peek[:n]) {
+		result.extension = ".webm"
+		result.contentType = "video/webm"
+		return result, nil
+	}
+	result.extension = ".mp4"
+	result.contentType = "video/mp4"
+	return result, nil
+}
+
 func jpegOrientation(raw []byte) int {
 	metadata, err := exif.Decode(bytes.NewReader(raw))
 	if err != nil {

internal/web/post_images_test.go

@@ -12,6 +12,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 
@@ -56,6 +57,47 @@ func TestPreparePostImage(t *testing.T) {
 	}
 }
 
+func TestPreparePostVideo(t *testing.T) {
+	t.Parallel()
+
+	mp4 := tinyMP4()
+	prepared, err := preparePostVideoHeader(t, "clip.mp4", mp4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer prepared.body.Close()
+	got, err := io.ReadAll(prepared.body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if prepared.extension != ".mp4" || prepared.contentType != "video/mp4" ||
+		prepared.size != int64(len(mp4)) || !bytes.Equal(got, mp4) {
+		t.Fatalf("prepared MP4 = %+v len(body)=%d", prepared, len(got))
+	}
+
+	webm := tinyWebM()
+	prepared, err = preparePostVideoHeader(t, "clip.webm", webm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer prepared.body.Close()
+	if prepared.extension != ".webm" || prepared.contentType != "video/webm" ||
+		prepared.size != int64(len(webm)) {
+		t.Fatalf("prepared WebM = %+v", prepared)
+	}
+	if _, err := preparePostVideoHeader(t, "empty.mp4", nil); err == nil {
+		t.Fatal("empty video unexpectedly succeeded")
+	}
+	if _, err := preparePostVideoHeader(t, "notes.txt", []byte("not a video")); err == nil {
+		t.Fatal("text video upload unexpectedly succeeded")
+	}
+	_, err = preparePostVideoHeader(t, "too-large.mp4", make([]byte, postVideoMaxFileBytes+1))
+	var requestErr *postImageRequestError
+	if !errors.As(err, &requestErr) || requestErr.status != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized video error = %v, want 413 request error", err)
+	}
+}
+
 func TestOrientPostImage(t *testing.T) {
 	t.Parallel()
 
@@ -203,6 +245,87 @@ func TestPostImageMultipartLifecycle(t *testing.T) {
 	}
 }
 
+func TestPostVideoMultipartLifecycle(t *testing.T) {
+	t.Parallel()
+
+	blobs := &recordingImageBlob{}
+	srv, mem := newTestServer(t, Config{Blob: blobs})
+	handler := srv.Handler()
+	homeowner := seedUser(t, mem, uniq("video"), "hunter22", store.RoleUser)
+	cookies := loginUser(t, handler, homeowner.Username, "hunter22")
+	csrf := csrfForCookies(t, handler, cookies)
+
+	rec := multipartPost(t, handler, "/submit", map[string][]string{
+		"_csrf":             {csrf},
+		"title":             {"Valve clip"},
+		"body":              {"A photo and a video."},
+		"city":              {"Oakland"},
+		"image_description": {"Still", "Walkthrough"},
+	}, []multipartTestFile{
+		{name: "still.png", body: solidPNG(t, 40, 20)},
+		{name: "walk.mp4", body: tinyMP4()},
+	}, cookies)
+	if rec.Code != http.StatusSeeOther {
+		t.Fatalf("root video upload status = %d: %s", rec.Code, rec.Body.String())
+	}
+	roots, err := mem.ListRootPosts(context.Background(), pacific.Today(), homeowner.ID)
+	if err != nil || len(roots) != 1 {
+		t.Fatalf("roots = %+v, %v", roots, err)
+	}
+	root, err := mem.GetPost(context.Background(), roots[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(root.Images) != 2 ||
+		root.Images[0].Kind != store.MediaKindImage ||
+		root.Images[1].Kind != store.MediaKindVideo ||
+		root.Images[1].Description != "Walkthrough" ||
+		!strings.HasPrefix(root.Images[1].ObjectKey, "post-videos/") {
+		t.Fatalf("root media = %+v", root.Images)
+	}
+	clip := tinyMP4()
+	var streamed recordedImageUpload
+	for _, upload := range blobs.recordedUploads() {
+		if strings.HasPrefix(upload.key, "post-videos/") {
+			streamed = upload
+			break
+		}
+	}
+	if streamed.size != int64(len(clip)) || !bytes.Equal(streamed.body, clip) {
+		t.Fatalf("streamed video upload = %+v", streamed)
+	}
+
+	rec = multipartPost(t, handler, "/posts/"+root.ID+"/edit", map[string][]string{
+		"_csrf":                      {csrf},
+		"body":                       {"Keep the clip."},
+		"existing_image_id":          {root.Images[1].ID},
+		"existing_image_description": {"Kept clip"},
+	}, nil, cookies)
+	if rec.Code != http.StatusSeeOther {
+		t.Fatalf("retain video status = %d: %s", rec.Code, rec.Body.String())
+	}
+	edited, err := mem.GetPost(context.Background(), root.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(edited.Images) != 1 || edited.Images[0].Kind != store.MediaKindVideo ||
+		edited.Images[0].Description != "Kept clip" {
+		t.Fatalf("retained video = %+v", edited.Images)
+	}
+
+	rec = multipartPost(t, handler, "/posts", map[string][]string{
+		"_csrf":     {csrf},
+		"parent_id": {root.ID},
+		"body":      {"Two clips."},
+	}, []multipartTestFile{
+		{name: "a.mp4", body: tinyMP4()},
+		{name: "b.mp4", body: tinyMP4()},
+	}, cookies)
+	if rec.Code != http.StatusBadRequest {
+		t.Fatalf("two-video status = %d, want 400", rec.Code)
+	}
+}
+
 func TestPostImageUploadCompensation(t *testing.T) {
 	t.Parallel()
 
@@ -318,6 +441,12 @@ func multipartPost(
 	return rec
 }
 
+func preparePostVideoHeader(t *testing.T, name string, body []byte) (preparedPostVideo, error) {
+	t.Helper()
+	header := multipartFileHeader(t, name, body)
+	return preparePostVideo(header)
+}
+
 func preparePostImageHeader(t *testing.T, name string, body []byte) (preparedPostImage, error) {
 	t.Helper()
 	var requestBody bytes.Buffer
@@ -341,6 +470,41 @@ func preparePostImageHeader(t *testing.T, name string, body []byte) (preparedPos
 	return preparePostImage(req.MultipartForm.File["images"][0])
 }
 
+func multipartFileHeader(t *testing.T, name string, body []byte) *multipart.FileHeader {
+	t.Helper()
+	var requestBody bytes.Buffer
+	writer := multipart.NewWriter(&requestBody)
+	part, err := writer.CreateFormFile("images", name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := part.Write(body); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	req := httptest.NewRequest(http.MethodPost, "/posts", &requestBody)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	if err := req.ParseMultipartForm(postImageMultipartMemory); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = req.MultipartForm.RemoveAll() })
+	return req.MultipartForm.File["images"][0]
+}
+
+func tinyMP4() []byte {
+	body := make([]byte, 16)
+	body[3] = 16
+	copy(body[4:], "ftypisom")
+	copy(body[12:], "isom")
+	return body
+}
+
+func tinyWebM() []byte {
+	return []byte{0x1a, 0x45, 0xdf, 0xa3, 0x01, 0x00, 0x00, 0x00}
+}
+
 func solidPNG(t *testing.T, width, height int) []byte {
 	t.Helper()
 	img := image.NewNRGBA(image.Rect(0, 0, width, height))
@@ -374,6 +538,7 @@ func solidJPEG(t *testing.T, width, height int) []byte {
 type recordedImageUpload struct {
 	key         string
 	contentType string
+	size        int64
 	body        []byte
 }
 
@@ -401,6 +566,7 @@ func (b *recordingImageBlob) Upload(_ context.Context, object blob.FileUpload) (
 	b.uploads = append(b.uploads, recordedImageUpload{
 		key:         object.Key,
 		contentType: object.ContentType,
+		size:        object.Size,
 		body:        body,
 	})
 	return "https://cdn.example/" + object.Key, nil
@@ -414,9 +580,13 @@ func (b *recordingImageBlob) Delete(_ context.Context, key string) error {
 }
 
 func (b *recordingImageBlob) uploadCount() int {
+	return len(b.recordedUploads())
+}
+
+func (b *recordingImageBlob) recordedUploads() []recordedImageUpload {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	return len(b.uploads)
+	return append([]recordedImageUpload(nil), b.uploads...)
 }
 
 func (b *recordingImageBlob) deletedKeys() []string {