~evanj/cms

630f31ab0b5b813e13a9356912c107be371a59e1 — Evan J 4 years ago 7c487cb master
Feat(file.go): Upload files concurrently complete. Fixed bug in E3 path
building (sometime left 'e' was being cut from strings.TrimLeft).
4 files changed, 105 insertions(+), 83 deletions(-)

M TODO
M internal/c/content/content.go
M internal/c/file/file.go
M pkg/e3/e3.go
M TODO => TODO +3 -3
@@ 3,10 3,10 @@ Testing: 100% happy path and 80% total
Write dynamic content
Depth option on APIs
Forgot password
Warn & delete excess users/spaces on downgrade.
Warn & delete excess users/spaces on downgrade
Cache: org?, invite?, role?, hook 
Admin: change role level of users.
Provide description of different roles (new page).
Admin: change role level of users
Provide description of different roles (new page)
Add more value types: number, time, datetime (already have time), timestamp?, 
  json


M internal/c/content/content.go => internal/c/content/content.go +101 -78
@@ 5,6 5,7 @@ import (
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"strconv"


@@ 80,18 81,71 @@ func New(c *c.Controller, log *log.Logger, db dber, e3 E3er, hook Hooker, baseUR
	}
}

func (c *Content) upload(ctx context.Context, filename string, file io.Reader, u user.User) (string, error) {
	raw, err := c.e3.Upload(ctx, false, filename, file)
type fileSet struct {
	key, url string
}

func upload(ctx context.Context, e3 E3er, r *http.Request, file multipart.File, header *multipart.FileHeader, baseURL, key string, c chan fileSet, e chan error) {
	raw, err := e3.Upload(ctx, false, header.Filename, file)
	if err != nil {
		return "", err
		e <- fmt.Errorf("failed to upload file: %w", err)
		return
	}

	val, err := url.Parse(raw)
	if err != nil {
		return "", err
		e <- fmt.Errorf("failed to upload file: %w", err)
		return
	}

	url := baseURL + "/file" + val.Path

	c <- fileSet{key, url}
}

func uploadFileSets(e3 E3er, baseURL string, r *http.Request, u user.User) ([]fileSet, error) {
	_, _, _ = r.FormFile("") // Dummy read, loads internal r.MulitpartForm state.

	if r.MultipartForm == nil {
		// No work to do.
		return []fileSet{}, nil
	}

	return c.baseURL + "/file" + val.Path, nil
	var (
		fs  []fileSet
		ctx = r.Context()
		l   = len(r.MultipartForm.File)
		c   = make(chan fileSet, l)
		e   = make(chan error, l)
	)

	if l < 1 {
		// No work to do.
		return []fileSet{}, nil
	}

	for key := range r.MultipartForm.File {
		file, header, err := r.FormFile(key)
		if err != nil {
			return []fileSet{}, err
		}
		go upload(ctx, e3, r, file, header, baseURL, key, c, e)
	}

	for {
		select {
		case <-ctx.Done():
			return []fileSet{}, ctx.Err()
		case err := <-e:
			return []fileSet{}, err
		case f := <-c:
			fs = append(fs, f)
			if len(fs) == l {
				// Done.
				return fs, nil
			}
		}
	}
}

func (c *Content) tree(w http.ResponseWriter, r *http.Request, spaceID, contenttypeID, contentID string) (user.User, space.Space, contenttype.ContentType, content.Content, error) {


@@ 164,36 218,23 @@ func (c *Content) create(w http.ResponseWriter, r *http.Request) {
		})
	}

	// TODO: Upload concurrently.
	_, _, _ = r.FormFile("") // Dummy read, loads internal r.MulitpartForm state.
	if r.MultipartForm != nil {
		for key := range r.MultipartForm.File {
			file, header, err := r.FormFile(key)
			if err != nil {
				c.Error(w, r, http.StatusInternalServerError, fmt.Errorf("failed to retreive file: %w", err))
				return
			}

			url, err := c.upload(r.Context(), header.Filename, file, user)
			if err != nil {
				c.Error(w, r, http.StatusInternalServerError, fmt.Errorf("failed to upload file: %w", err))
				return
			}

			parts := strings.Split(key, "-")
			if len(parts) < 2 {
				c.Error(w, r, http.StatusInternalServerError, errors.New("invalid name field for value"))
				return
			}

			typ, name := parts[0], parts[1]

			params = append(params, db.ContentNewParam{
				Type:  typ,
				Name:  name,
				Value: url,
			})
	// Upload files concurrently.
	fileSets, err := uploadFileSets(c.e3, c.baseURL, r, user)
	if err != nil {
		c.Error(w, r, http.StatusInternalServerError, err)
		return
	}
	for _, fileSet := range fileSets {
		parts := strings.Split(fileSet.key, "-")
		if len(parts) < 2 {
			c.Error(w, r, http.StatusInternalServerError, errors.New("invalid name field for value"))
			return
		}
		params = append(params, db.ContentNewParam{
			Type:  parts[0],
			Name:  parts[1],
			Value: fileSet.url,
		})
	}

	content, err := c.db.ContentNew(r.Context(), user, space, ct, params)


@@ 318,56 359,38 @@ func (c *Content) update(w http.ResponseWriter, r *http.Request) {
		})
	}

	// TODO: Upload concurrently.
	_, _, _ = r.FormFile("") // Dummy read, loads internal r.MulitpartForm state.
	if r.MultipartForm != nil {
		for key := range r.MultipartForm.File {
			file, header, err := r.FormFile(key)
			if err != nil {
				c.Error(w, r, http.StatusInternalServerError, fmt.Errorf("failed to retreive file: %w", err))
				return
			}

			url, err := c.upload(r.Context(), header.Filename, file, user)
			if err != nil {
				c.Error(w, r, http.StatusInternalServerError, fmt.Errorf("failed to upload file: %w", err))
				return
			}

			// Check if we're update image value.
			if strings.Contains(key, "value_update_") {

				parts := strings.Split(strings.ReplaceAll(key, "value_update_", ""), "-")
				if len(parts) < 2 {
					c.Error(w, r, http.StatusInternalServerError, errors.New("invalid name field for value"))
					return
				}

				_, id := parts[0], parts[1]

				updateParams = append(updateParams, db.ContentUpdateParam{
					ID:    id,
					Type:  valuetype.File,
					Value: url,
				})

				continue
			}

			parts := strings.Split(key, "-")
	// Upload files concurrently.
	fileSets, err := uploadFileSets(c.e3, c.baseURL, r, user)
	if err != nil {
		c.Error(w, r, http.StatusInternalServerError, err)
		return
	}
	for _, fileSet := range fileSets {
		// Check if we're update file value.
		if strings.Contains(fileSet.key, "value_update_") {
			parts := strings.Split(strings.ReplaceAll(fileSet.key, "value_update_", ""), "-")
			if len(parts) < 2 {
				c.Error(w, r, http.StatusInternalServerError, errors.New("invalid name field for value"))
				return
			}

			typ, name := parts[0], parts[1]

			newParams = append(newParams, db.ContentNewParam{
				Type:  typ,
				Name:  name,
				Value: url,
			updateParams = append(updateParams, db.ContentUpdateParam{
				ID:    parts[1], // ID
				Type:  valuetype.File,
				Value: fileSet.url,
			})
			continue
		}
		// This is a new file value.
		parts := strings.Split(fileSet.key, "-")
		if len(parts) < 2 {
			c.Error(w, r, http.StatusInternalServerError, errors.New("invalid name field for value"))
			return
		}
		newParams = append(newParams, db.ContentNewParam{
			Type:  parts[0],
			Name:  parts[1],
			Value: fileSet.url,
		})
	}

	content, err = c.db.ContentUpdate(r.Context(), user, space, ct, content, newParams, updateParams)

M internal/c/file/file.go => internal/c/file/file.go +1 -1
@@ 40,7 40,7 @@ func (f *File) ServeHTTP(w http.ResponseWriter, r *http.Request) {
		return
	}

	full := strings.TrimRight(f.e3.URL(), "api") + strings.TrimLeft(r.URL.Path, "/file") // TODO: Cleanup, this is hacky.
	full := strings.TrimRight(f.e3.URL(), "api") + strings.TrimPrefix(r.URL.Path, "/file") // TODO: Cleanup, this is hacky.
	bytes, err := f.e3.Proxy(r.Context(), full)
	if err != nil {
		f.ErrorString(w, r, http.StatusInternalServerError, "failed to serve file")

M pkg/e3/e3.go => pkg/e3/e3.go +0 -1
@@ 99,7 99,6 @@ func (e3 E3) Proxy(ctx context.Context, objectURL string) ([]byte, error) {
	if err != nil {
		return []byte{}, err
	}

	req.SetBasicAuth(e3.user, e3.pass)

	resp, err := e3.client.Do(req)