From 6a6ad774904594689a0f41d85e0e85ac04703380 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Sun, 29 Oct 2017 20:38:19 +0100 Subject: [PATCH] Initial commit --- .gitignore | 15 +++ .vscode/launch.json | 21 +++++ README.md | 32 +++++++ docker-compose.yml | 9 ++ handlers/handlers.go | 193 ++++++++++++++++++++++++++++++++++++++ handlers/handlers_test.go | 138 +++++++++++++++++++++++++++ main.go | 32 +++++++ run.sh | 3 + store/store.go | 191 +++++++++++++++++++++++++++++++++++++ store/store_test.go | 118 +++++++++++++++++++++++ 10 files changed, 752 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 handlers/handlers.go create mode 100644 handlers/handlers_test.go create mode 100644 main.go create mode 100644 run.sh create mode 100644 store/store.go create mode 100644 store/store_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..67a9dee --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ +main.db \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..88f947a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch", + "type": "go", + "request": "launch", + "mode": "debug", + "remotePath": "", + "port": 2345, + "host": "127.0.0.1", + "program": "${fileDirname}", + "env": {}, + "args": [], + "showLog": true + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..313b935 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Golang URL Shorter using BoltDB + +## Features: + +- URL Shorting with visit counting +- delete a URL +- Authorization via tokens +- Storing using BoltDB + +## Installation + +### Standard + +```bash +go get -v ./... +go run -v main.go +``` +### Docker Compose + +Only execute the [docker-compose.yml](docker-compose.yml) and adjust the enviroment variables to your needs. + +## [ShareX](https://github.com/ShareX/ShareX) Configuration + +## TODOs + +- Unit tests +- code refactoring +- enviroment or configuration file integration +- github publishing +- authentification +- deletion + - ShareX example \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a3d9013 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +shorter: + image: golang + restart: always + command: ./run.sh + working_dir: /go/src/server/workspace + volumes: + - ./:/go/src/server/workspace + ports: + - 8080:8080 diff --git a/handlers/handlers.go b/handlers/handlers.go new file mode 100644 index 0000000..f77707c --- /dev/null +++ b/handlers/handlers.go @@ -0,0 +1,193 @@ +package handlers + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + + "github.com/julienschmidt/httprouter" + "github.com/maxibanki/golang-url-shorter/store" +) + +// Handler holds the funcs and attributes for the +// http communication +type Handler struct { + addr string + store store.Store + server *http.Server +} + +// URLUtil is used to help in- and outgoing requests for json +// un- and marshalling +type URLUtil struct { + URL string +} + +// New initializes the http handlers +func New(addr string, store store.Store) *Handler { + h := &Handler{ + addr: addr, + store: store, + } + router := h.handlers() + h.server = &http.Server{Addr: h.addr, Handler: router} + return h +} + +func (h *Handler) handlers() *httprouter.Router { + router := httprouter.New() + router.POST("/api/v1/create", h.handleCreate) + router.POST("/api/v1/info", h.handleInfo) + router.GET("/:id", h.handleAccess) + return router +} + +// handleCreate handles requests to create an entry +func (h *Handler) handleCreate(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + contentType := r.Header.Get("Content-Type") + switch contentType { + case "application/json": + h.handleCreateJSON(w, r) + break + case "application/x-www-form-urlencoded": + h.handleCreateForm(w, r) + break + default: + if strings.Contains(contentType, "multipart/form-data;") { + h.handleCreateMultipartForm(w, r) + return + } + log.Printf("could not detect Content-Type: %s", contentType) + } +} + +func (h *Handler) handleCreateJSON(w http.ResponseWriter, r *http.Request) { + var req URLUtil + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, fmt.Sprintf("could not decode JSON: %v", err), http.StatusBadRequest) + return + } + id, err := h.store.CreateEntry(req.URL, r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + req.URL = h.generateURL(r, id) + err = json.NewEncoder(w).Encode(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} + +func (h *Handler) handleCreateMultipartForm(w http.ResponseWriter, r *http.Request) { + err := r.ParseMultipartForm(1048576) + if err != nil { + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + if _, ok := r.MultipartForm.Value["URL"]; !ok { + http.Error(w, "URL key does not exist", http.StatusBadRequest) + return + } + id, err := h.store.CreateEntry(r.MultipartForm.Value["URL"][0], r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + var req URLUtil + req.URL = h.generateURL(r, id) + err = json.NewEncoder(w).Encode(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} + +func (h *Handler) handleCreateForm(w http.ResponseWriter, r *http.Request) { + err := r.ParseForm() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + id, err := h.store.CreateEntry(r.PostFormValue("URL"), r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + var req URLUtil + req.URL = h.generateURL(r, id) + err = json.NewEncoder(w).Encode(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} + +func (h *Handler) generateURL(r *http.Request, id string) string { + protocol := "http" + if r.TLS != nil { + protocol = "https" + } + return fmt.Sprintf("%s://%s/%s", protocol, r.Host, id) +} + +// handleInfo is the http handler for getting the infos +func (h *Handler) handleInfo(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + var req struct { + ID string + } + if r.Body == nil { + http.Error(w, "invalid request, body is nil", http.StatusBadRequest) + return + } + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + raw, err := h.store.GetEntryByIDRaw(req.ID) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + + return + } + w.Header().Add("Content-Type", "application/json") + w.Write(raw) +} + +// handleAccess handles the access for incoming requests +func (h *Handler) handleAccess(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + id := p.ByName("id") + entry, err := h.store.GetEntryByID(id) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + err = h.store.IncreaseVisitCounter(id) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + http.Redirect(w, r, entry.URL, http.StatusTemporaryRedirect) +} + +// Listen starts the http server +func (h *Handler) Listen() error { + return h.server.ListenAndServe() +} + +// Stop stops the http server and the closes the db gracefully +func (h *Handler) Stop() error { + err := h.store.Close() + if err != nil { + return err + } + return h.server.Shutdown(context.Background()) +} diff --git a/handlers/handlers_test.go b/handlers/handlers_test.go new file mode 100644 index 0000000..5cf151e --- /dev/null +++ b/handlers/handlers_test.go @@ -0,0 +1,138 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/maxibanki/golang-url-shorter/store" + "github.com/pkg/errors" +) + +const ( + baseURL = "http://myshorter" + testingDBName = "main.db" +) + +var server *httptest.Server + +func TestCreateEntryJSON(t *testing.T) { + tt := []struct { + name string + ignoreResponse bool + contentType string + response string + responseBody URLUtil + statusCode int + }{ + { + name: "body is nil", + response: "invalid request, body is nil", + statusCode: http.StatusBadRequest, + contentType: "appication/json", + ignoreResponse: true, + }, + { + name: "short URL generation", + responseBody: URLUtil{ + URL: "https://www.google.de/", + }, + statusCode: http.StatusOK, + contentType: "appication/json", + }, + } + cleanup, err := getBackend() + if err != nil { + t.Fatalf("could not create backend: %v", err) + } + defer cleanup() + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + // build body for the create URL http request + var reqBody *bytes.Buffer + if tc.responseBody.URL != "" { + json, err := json.Marshal(tc.responseBody) + if err != nil { + t.Fatalf("could not marshal json: %v", err) + } + reqBody = bytes.NewBuffer(json) + } else { + reqBody = bytes.NewBuffer(nil) + } + resp, err := http.Post(server.URL+"/api/v1/create", "application/json", reqBody) + if err != nil { + t.Fatalf("could not create post request: %v", err) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("could not read response: %v", err) + } + body = bytes.TrimSpace(body) + if tc.ignoreResponse { + return + } + if resp.StatusCode != tc.statusCode { + t.Errorf("expected status %d; got %d", tc.statusCode, resp.StatusCode) + } + if tc.response != "" { + if string(body) != string(tc.response) { + t.Fatalf("expected body: %s; got: %s", tc.response, body) + } + } + var parsed URLUtil + err = json.Unmarshal(body, &parsed) + if err != nil { + t.Fatalf("could not unmarshal data: %v", err) + } + t.Run("test if shorted URL is correct", func(t *testing.T) { + testRedirect(t, parsed.URL, tc.responseBody.URL) + }) + }) + } +} + +func testRedirect(t *testing.T, shortURL, longURL string) { + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, // don't follow redirects + } + u, err := url.Parse(shortURL) + if err != nil { + t.Fatalf("could not parse shorted URL: %v", err) + } + respShort, err := client.Do(&http.Request{ + URL: u, + }) + if err != nil { + t.Fatalf("could not do http request to shorted URL: %v", err) + } + if respShort.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("expected status code: %d; got: %d", http.StatusTemporaryRedirect, respShort.StatusCode) + } + if respShort.Header.Get("Location") != longURL { + t.Fatalf("redirect URL is not correct") + } +} + +func getBackend() (func(), error) { + store, err := store.New(testingDBName, 4) + if err != nil { + return nil, errors.Wrap(err, "could not create store") + } + handler := New(":8080", *store) + if err != nil { + return nil, errors.Wrap(err, "could not create handler") + } + server = httptest.NewServer(handler.handlers()) + return func() { + server.Close() + handler.Stop() + os.Remove(testingDBName) + }, nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..e834fef --- /dev/null +++ b/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + "os" + "os/signal" + + "github.com/maxibanki/golang-url-shorter/handlers" + "github.com/maxibanki/golang-url-shorter/store" +) + +func main() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt) + store, err := store.New("main.db", 4) + if err != nil { + log.Fatalf("could not create store: %v", err) + } + handler := handlers.New(":8080", *store) + go func() { + err := handler.Listen() + if err != nil { + log.Fatalf("could not listen to http handlers: %v", err) + } + }() + <-stop + log.Println("Shutting down...") + err = handler.Stop() + if err != nil { + log.Printf("failed to stop the handlers: %v", err) + } +} diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..313f4e2 --- /dev/null +++ b/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +go get -v ./... +go run -v main.go diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..e4ecdab --- /dev/null +++ b/store/store.go @@ -0,0 +1,191 @@ +package store + +import ( + "encoding/json" + "math/rand" + "time" + + "github.com/asaskevich/govalidator" + "github.com/boltdb/bolt" + "github.com/pkg/errors" +) + +// Store holds internal funcs and vars about the store +type Store struct { + db *bolt.DB + bucketName []byte + idLength uint +} + +// Entry is the data set which is stored in the DB as JSON +type Entry struct { + URL string + VisitCount int + RemoteAddr string + CreatedOn time.Time + LastVisit time.Time +} + +// ErrNoEntryFound is returned when no entry to a id is found +var ErrNoEntryFound = errors.New("no entry found") + +// ErrNoValidURL is returned when the URL is not valid +var ErrNoValidURL = errors.New("no valid URL") + +// ErrGeneratingTriesFailed is returned when the 10 tries to generate an id failed +var ErrGeneratingTriesFailed = errors.New("could not generate unique id which doesn't exist in the db") + +// ErrIDIsEmpty is returned when the given ID is empty +var ErrIDIsEmpty = errors.New("id is empty") + +// New initializes the store with the db +func New(dbName string, idLength uint) (*Store, error) { + db, err := bolt.Open(dbName, 0644, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return nil, errors.Wrap(err, "could not open bolt DB database") + } + bucketName := []byte("shortUrlBkt") + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(bucketName) + return err + }) + if err != nil { + return nil, err + } + return &Store{ + db: db, + idLength: idLength, + bucketName: bucketName, + }, nil +} + +// GetEntryByID returns a unmarshalled entry of the db by a given ID +func (s *Store) GetEntryByID(id string) (*Entry, error) { + if id == "" { + return nil, ErrIDIsEmpty + } + raw, err := s.GetEntryByIDRaw(id) + if err != nil { + return nil, err + } + var entry *Entry + err = json.Unmarshal(raw, &entry) + return entry, err +} + +// IncreaseVisitCounter increments the visit counter of an entry +func (s *Store) IncreaseVisitCounter(id string) error { + entry, err := s.GetEntryByID(id) + if err != nil { + return err + } + entry.VisitCount++ + entry.LastVisit = time.Now() + raw, err := json.Marshal(entry) + if err != nil { + return err + } + err = s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(s.bucketName) + err := bucket.Put([]byte(id), raw) + if err != nil { + return errors.Wrap(err, "could not put data into bucket") + } + return nil + }) + return err +} + +// GetEntryByIDRaw returns the raw data (JSON) of a data set +func (s *Store) GetEntryByIDRaw(id string) ([]byte, error) { + var raw []byte + err := s.db.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket(s.bucketName) + raw = bucket.Get([]byte(id)) + if raw == nil { + return ErrNoEntryFound + } + return nil + }) + return raw, err +} + +// CreateEntry creates a new record and returns his short id +func (s *Store) CreateEntry(URL, remoteAddr string) (string, error) { + if !govalidator.IsURL(URL) { + return "", ErrNoValidURL + } + // try it 10 times to make a short URL + for i := 1; i <= 10; i++ { + id, err := s.createEntry(URL, remoteAddr) + if err != nil { + continue + } + return id, nil + } + return "", ErrGeneratingTriesFailed +} + +// checkExistens returns true if a entry with a given ID +// exists and false if not +func (s *Store) checkExistence(id string) bool { + raw, err := s.GetEntryByIDRaw(id) + if err != nil && err != ErrNoEntryFound { + return true + } + if raw != nil { + return true + } + return false +} + +// createEntry creates a new entry +func (s *Store) createEntry(URL, remoteAddr string) (string, error) { + id := generateRandomString(s.idLength) + + exists := s.checkExistence(id) + if !exists { + raw, err := json.Marshal(Entry{ + URL: URL, + RemoteAddr: remoteAddr, + CreatedOn: time.Now(), + }) + if err != nil { + return "", err + } + return id, s.createEntryRaw([]byte(id), raw) + } + return "", ErrGeneratingTriesFailed +} + +// createEntryRaw creates a entry with the given key value pair +func (s *Store) createEntryRaw(key, value []byte) error { + err := s.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(s.bucketName) + raw := bucket.Get(key) + if raw != nil { + return errors.New("entry value is not empty") + } + err := bucket.Put(key, value) + if err != nil { + return errors.Wrap(err, "could not put data into bucket") + } + return nil + }) + return err +} + +// Close closes the bolt db database +func (s *Store) Close() error { + return s.db.Close() +} + +// generateRandomString generates a random string with an predefined length +func generateRandomString(length uint) string { + letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + b := make([]rune, length) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} diff --git a/store/store_test.go b/store/store_test.go new file mode 100644 index 0000000..6288948 --- /dev/null +++ b/store/store_test.go @@ -0,0 +1,118 @@ +package store + +import ( + "os" + "testing" +) + +const ( + testingDBName = "test.db" +) + +func TestGenerateRandomString(t *testing.T) { + tt := []struct { + name string + length uint + }{ + {"fourtytwo long", 42}, + {"sixteen long", 16}, + {"eighteen long", 19}, + {"zero long", 0}, + {"onehundretseventyfive long", 157}, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + rnd := generateRandomString(tc.length) + if len(rnd) != int(tc.length) { + t.Fatalf("length of %s random string is %d not the expected one: %d", tc.name, len(rnd), tc.length) + } + }) + } +} + +func TestNewStore(t *testing.T) { + t.Run("create store without file name provided", func(r *testing.T) { + _, err := New("", 4) + if err.Error() != "could not open bolt DB database: open : The system cannot find the file specified." { + t.Fatalf("unexpected error: %v", err) + } + }) + t.Run("create store with correct arguments", func(r *testing.T) { + store, err := New(testingDBName, 4) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + cleanup(store) + }) +} + +func TestCreateEntry(t *testing.T) { + store, err := New(testingDBName, 1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer cleanup(store) + _, err = store.CreateEntry("", "") + if err != ErrNoValidURL { + t.Fatalf("unexpected error: %v", err) + } + for i := 1; i <= 100; i++ { + _, err := store.CreateEntry("https://golang.org/", "") + if err != nil && err != ErrGeneratingTriesFailed { + t.Fatalf("unexpected error during creating entry: %v", err) + } + } +} + +func TestGetEntryByID(t *testing.T) { + store, err := New(testingDBName, 1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer cleanup(store) + _, err = store.GetEntryByID("something that not exists") + if err != ErrNoEntryFound { + t.Fatalf("could not get expected '%v' error: %v", ErrNoEntryFound, err) + } + _, err = store.GetEntryByID("") + if err != ErrIDIsEmpty { + t.Fatalf("could not get expected '%v' error: %v", ErrIDIsEmpty, err) + } +} + +func TestIncreaseVisitCounter(t *testing.T) { + store, err := New(testingDBName, 4) + if err != nil { + t.Fatalf("could not create store: %v", err) + } + defer cleanup(store) + id, err := store.CreateEntry("https://golang.org/", "") + if err != nil { + t.Fatalf("could not create entry: %v", err) + } + entryBeforeInc, err := store.GetEntryByID(id) + if err != nil { + t.Fatalf("could not get entry by id: %v", err) + } + err = store.IncreaseVisitCounter(id) + if err != nil { + t.Fatalf("could not increase visit counter %v", err) + } + entryAfterInc, err := store.GetEntryByID(id) + if err != nil { + t.Fatalf("could not get entry by id: %v", err) + } + if entryBeforeInc.VisitCount+1 != entryAfterInc.VisitCount { + t.Fatalf("the increasement was not successful, the visit count is not correct") + } + err = store.IncreaseVisitCounter("") + if err != ErrIDIsEmpty { + t.Fatalf("could not get expected '%v' error: %v", ErrIDIsEmpty, err) + } +} + +func cleanup(s *Store) { + s.Close() + os.Remove(testingDBName) +}