From 071c94b2d71cf0d6c163e70445d2be206abe46f0 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Sun, 24 Dec 2017 14:13:34 +0100 Subject: [PATCH] Abstracted boltdb as a storage interface (fix #60) --- handlers/auth_test.go | 4 +- handlers/handlers.go | 6 +- handlers/public.go | 24 +-- handlers/public_test.go | 9 +- main.go | 9 +- store/store.go | 276 -------------------------------- store/util.go | 70 -------- stores/boltdb/bolt.go | 193 ++++++++++++++++++++++ stores/shared/shared.go | 46 ++++++ stores/sqlite/sqlite.go | 1 + stores/store.go | 189 ++++++++++++++++++++++ {store => stores}/store_test.go | 60 ++----- util/config.go | 2 +- 13 files changed, 472 insertions(+), 417 deletions(-) delete mode 100644 store/store.go delete mode 100644 store/util.go create mode 100644 stores/boltdb/bolt.go create mode 100644 stores/shared/shared.go create mode 100644 stores/sqlite/sqlite.go create mode 100644 stores/store.go rename {store => stores}/store_test.go (65%) diff --git a/handlers/auth_test.go b/handlers/auth_test.go index 23303b0..fa21c28 100644 --- a/handlers/auth_test.go +++ b/handlers/auth_test.go @@ -12,7 +12,7 @@ import ( jwt "github.com/dgrijalva/jwt-go" "github.com/gin-gonic/gin" "github.com/maxibanki/golang-url-shortener/handlers/auth" - "github.com/maxibanki/golang-url-shortener/store" + "github.com/maxibanki/golang-url-shortener/stores" "github.com/maxibanki/golang-url-shortener/util" "github.com/pkg/errors" ) @@ -38,7 +38,7 @@ func TestCreateBackend(t *testing.T) { if err := util.ReadInConfig(); err != nil { t.Fatalf("could not reload config file: %v", err) } - store, err := store.New() + store, err := stores.New() if err != nil { t.Fatalf("could not create store: %v", err) } diff --git a/handlers/handlers.go b/handlers/handlers.go index 47942b5..68c2f0b 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -11,7 +11,7 @@ import ( "github.com/gin-gonic/gin" "github.com/maxibanki/golang-url-shortener/handlers/tmpls" - "github.com/maxibanki/golang-url-shortener/store" + "github.com/maxibanki/golang-url-shortener/stores" "github.com/maxibanki/golang-url-shortener/util" "github.com/pkg/errors" ) @@ -19,7 +19,7 @@ import ( // Handler holds the funcs and attributes for the // http communication type Handler struct { - store store.Store + store stores.Store engine *gin.Engine providers []string } @@ -28,7 +28,7 @@ type Handler struct { var DoNotPrivateKeyChecking = false // New initializes the http handlers -func New(store store.Store) (*Handler, error) { +func New(store stores.Store) (*Handler, error) { if !util.GetConfig().EnableDebugMode { gin.SetMode(gin.ReleaseMode) } diff --git a/handlers/public.go b/handlers/public.go index 23a0a60..71b6d9b 100644 --- a/handlers/public.go +++ b/handlers/public.go @@ -8,11 +8,12 @@ import ( "net/http" "net/url" "runtime" + "strings" "time" "github.com/gin-gonic/gin" "github.com/maxibanki/golang-url-shortener/handlers/auth" - "github.com/maxibanki/golang-url-shortener/store" + "github.com/maxibanki/golang-url-shortener/stores/shared" "github.com/maxibanki/golang-url-shortener/util" "golang.org/x/crypto/bcrypt" ) @@ -40,8 +41,8 @@ func (h *Handler) handleLookup(c *gin.Context) { return } if !h.oAuthPropertiesEquals(c, entry.OAuthID, entry.OAuthProvider) { - c.JSON(http.StatusOK, store.Entry{ - Public: store.EntryPublicData{ + c.JSON(http.StatusOK, shared.Entry{ + Public: shared.EntryPublicData{ URL: entry.Public.URL, }, }) @@ -54,11 +55,12 @@ func (h *Handler) handleLookup(c *gin.Context) { func (h *Handler) handleAccess(c *gin.Context) { id := c.Request.URL.Path[1:] entry, err := h.store.GetEntryAndIncrease(id) - if err == store.ErrNoEntryFound { - return - } else if err != nil { - http.Error(c.Writer, fmt.Sprintf("could not get and crease visitor counter: %v, ", err), http.StatusInternalServerError) - return + if err != nil { + if strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) { + return + http.Error(c.Writer, fmt.Sprintf("could not get and crease visitor counter: %v, ", err), http.StatusInternalServerError) + return + } } // No password set if len(entry.Password) == 0 { @@ -100,8 +102,8 @@ func (h *Handler) handleCreate(c *gin.Context) { return } user := c.MustGet("user").(*auth.JWTClaims) - id, delID, err := h.store.CreateEntry(store.Entry{ - Public: store.EntryPublicData{ + id, delID, err := h.store.CreateEntry(shared.Entry{ + Public: shared.EntryPublicData{ URL: data.URL, Expiration: data.Expiration, }, @@ -190,7 +192,7 @@ func (h *Handler) getURLOrigin(c *gin.Context) string { } func (h *Handler) registerVisitor(id string, c *gin.Context) { - h.store.RegisterVisit(id, store.Visitor{ + h.store.RegisterVisit(id, shared.Visitor{ IP: c.ClientIP(), Timestamp: time.Now(), Referer: c.GetHeader("Referer"), diff --git a/handlers/public_test.go b/handlers/public_test.go index a54be73..c6d65b9 100644 --- a/handlers/public_test.go +++ b/handlers/public_test.go @@ -3,6 +3,7 @@ package handlers import ( "bytes" "encoding/json" + "fmt" "io/ioutil" "net/http" "net/url" @@ -10,7 +11,8 @@ import ( "testing" "github.com/gin-gonic/gin" - "github.com/maxibanki/golang-url-shortener/store" + "github.com/maxibanki/golang-url-shortener/stores" + "github.com/maxibanki/golang-url-shortener/stores/shared" ) const testURL = "https://www.google.de/" @@ -50,7 +52,7 @@ func TestCreateEntry(t *testing.T) { }, statusCode: http.StatusBadRequest, contentType: "application/json; charset=utf-8", - response: gin.H{"error": store.ErrNoValidURL.Error()}, + response: gin.H{"error": stores.ErrNoValidURL.Error()}, ignoreResponse: true, }, } @@ -121,7 +123,7 @@ func TestHandleInfo(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Errorf("expected status %d; got %d", http.StatusOK, resp.StatusCode) } - var entry store.EntryPublicData + var entry shared.EntryPublicData if err = json.NewDecoder(resp.Body).Decode(&entry); err != nil { t.Fatalf("could not unmarshal data: %v", err) } @@ -272,6 +274,7 @@ func TestHandleDeletion(t *testing.T) { if err != nil { t.Fatalf("could not send visit request: %v", err) } + fmt.Println(body.URL) if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected status: %d; got: %d", http.StatusNotFound, resp.StatusCode) } diff --git a/main.go b/main.go index 5df13ee..e86def7 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,12 @@ import ( "os" "os/signal" - "github.com/shiena/ansicolor" - "github.com/sirupsen/logrus" - "github.com/maxibanki/golang-url-shortener/handlers" - "github.com/maxibanki/golang-url-shortener/store" + "github.com/maxibanki/golang-url-shortener/stores" "github.com/maxibanki/golang-url-shortener/util" "github.com/pkg/errors" + "github.com/shiena/ansicolor" + "github.com/sirupsen/logrus" ) func main() { @@ -36,7 +35,7 @@ func initShortener() (func(), error) { if util.GetConfig().EnableDebugMode { logrus.SetLevel(logrus.DebugLevel) } - store, err := store.New() + store, err := stores.New() if err != nil { return nil, errors.Wrap(err, "could not create store") } diff --git a/store/store.go b/store/store.go deleted file mode 100644 index 4c61169..0000000 --- a/store/store.go +++ /dev/null @@ -1,276 +0,0 @@ -// Package store provides support to interact with the entries -package store - -import ( - "bytes" - "crypto/hmac" - "crypto/sha512" - "encoding/json" - "path/filepath" - "strings" - "time" - - "github.com/maxibanki/golang-url-shortener/util" - "github.com/pborman/uuid" - "github.com/sirupsen/logrus" - "golang.org/x/crypto/bcrypt" - - "github.com/asaskevich/govalidator" - "github.com/boltdb/bolt" - "github.com/pkg/errors" -) - -// Store holds internal funcs and vars about the store -type Store struct { - db *bolt.DB - idLength int -} - -// Entry is the data set which is stored in the DB as JSON -type Entry struct { - OAuthProvider, OAuthID string - RemoteAddr string `json:",omitempty"` - DeletionURL string `json:",omitempty"` - Password []byte `json:",omitempty"` - Public EntryPublicData -} - -// Visitor is the entry which is stored in the visitors bucket -type Visitor struct { - IP, Referer, UserAgent string - Timestamp time.Time - UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm string `json:",omitempty"` -} - -// EntryPublicData is the public part of an entry -type EntryPublicData struct { - CreatedOn time.Time - LastVisit, Expiration *time.Time `json:",omitempty"` - VisitCount int - URL string -} - -// ErrNoEntryFound is returned when no entry to a id is found -var ErrNoEntryFound = errors.New("no entry found with this ID") - -// ErrNoValidURL is returned when the URL is not valid -var ErrNoValidURL = errors.New("the given URL is no valid URL") - -// ErrGeneratingIDFailed is returned when the 10 tries to generate an id failed -var ErrGeneratingIDFailed = errors.New("could not generate unique id, all ten tries failed") - -// ErrEntryIsExpired is returned when the entry is expired -var ErrEntryIsExpired = errors.New("entry is expired") - -var ( - shortedURLsBucket = []byte("shorted") - shortedIDsToUserBucket = []byte("shorted2Users") -) - -// New initializes the store with the db -func New() (*Store, error) { - db, err := bolt.Open(filepath.Join(util.GetConfig().DataDir, "main.db"), 0644, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return nil, errors.Wrap(err, "could not open bolt DB database") - } - err = db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(shortedURLsBucket) - return err - }) - if err != nil { - return nil, err - } - return &Store{ - db: db, - idLength: util.GetConfig().ShortedIDLength, - }, nil -} - -// GetEntryByID returns a unmarshalled entry of the db by a given ID -func (s *Store) GetEntryByID(id string) (*Entry, error) { - if id == "" { - return nil, ErrNoEntryFound - } - rawEntry, err := s.GetEntryByIDRaw(id) - if err != nil { - return nil, err - } - var entry *Entry - return entry, json.Unmarshal(rawEntry, &entry) -} - -// IncreaseVisitCounter increments the visit counter of an entry -func (s *Store) IncreaseVisitCounter(id string) error { - entry, err := s.GetEntryByID(id) - if err != nil { - return errors.Wrap(err, "could not get entry by ID") - } - entry.Public.VisitCount++ - currentTime := time.Now() - entry.Public.LastVisit = ¤tTime - raw, err := json.Marshal(entry) - if err != nil { - return err - } - return s.db.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket(shortedURLsBucket).Put([]byte(id), raw); err != nil { - return errors.Wrap(err, "could not put updated visitor count JSON into the bucket") - } - return nil - }) -} - -// GetEntryAndIncrease Increases the visitor count, checks -// if the URL is expired and returns the origin URL -func (s *Store) GetEntryAndIncrease(id string) (*Entry, error) { - entry, err := s.GetEntryByID(id) - if err != nil { - return nil, err - } - if entry.Public.Expiration != nil && time.Now().After(*entry.Public.Expiration) { - return nil, ErrEntryIsExpired - } - if err := s.IncreaseVisitCounter(id); err != nil { - return nil, errors.Wrap(err, "could not increase visitor counter") - } - return entry, nil -} - -// GetEntryByIDRaw returns the raw data (JSON) of a data set -func (s *Store) GetEntryByIDRaw(id string) ([]byte, error) { - var raw []byte - return raw, s.db.View(func(tx *bolt.Tx) error { - raw = tx.Bucket(shortedURLsBucket).Get([]byte(id)) - if raw == nil { - return ErrNoEntryFound - } - return nil - }) -} - -// CreateEntry creates a new record and returns his short id -func (s *Store) CreateEntry(entry Entry, givenID, password string) (string, []byte, error) { - entry.Public.URL = strings.Replace(entry.Public.URL, " ", "%20", -1) - if !govalidator.IsURL(entry.Public.URL) { - return "", nil, ErrNoValidURL - } - if password != "" { - var err error - entry.Password, err = bcrypt.GenerateFromPassword([]byte(password), 10) - if err != nil { - return "", nil, errors.Wrap(err, "could not generate bcrypt from password") - } - } - // try it 10 times to make a short URL - for i := 1; i <= 10; i++ { - id, delID, err := s.createEntry(entry, givenID) - if err != nil && givenID != "" { - return "", nil, err - } else if err != nil { - logrus.Debugf("Could not create entry: %v", err) - continue - } - return id, delID, nil - } - return "", nil, ErrGeneratingIDFailed -} - -// DeleteEntry deletes an Entry fully from the DB -func (s *Store) DeleteEntry(id string, givenHmac []byte) error { - mac := hmac.New(sha512.New, util.GetPrivateKey()) - if _, err := mac.Write([]byte(id)); err != nil { - return errors.Wrap(err, "could not write hmac") - } - if !hmac.Equal(mac.Sum(nil), givenHmac) { - return errors.New("hmac verification failed") - } - return s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(shortedURLsBucket) - if bucket.Get([]byte(id)) == nil { - return errors.New("entry already deleted") - } - if err := bucket.Delete([]byte(id)); err != nil { - return errors.Wrap(err, "could not delete entry") - } - if err := tx.DeleteBucket([]byte(id)); err != nil && err != bolt.ErrBucketNotFound && err != bolt.ErrBucketExists { - return errors.Wrap(err, "could not delte bucket") - } - uTsIDsBucket := tx.Bucket(shortedIDsToUserBucket) - return uTsIDsBucket.ForEach(func(k, v []byte) error { - if bytes.Equal(k, []byte(id)) { - return uTsIDsBucket.Delete(k) - } - return nil - }) - }) -} - -// RegisterVisit registers an new incoming request in the store -func (s *Store) RegisterVisit(id string, visitor Visitor) { - requestID := uuid.New() - logrus.WithFields(logrus.Fields{ - "ClientIP": visitor.IP, - "ID": id, - "RequestID": requestID, - }).Info("New redirect was registered...") - - err := s.db.Update(func(tx *bolt.Tx) error { - bucket, err := tx.CreateBucketIfNotExists([]byte(id)) - if err != nil { - return errors.Wrap(err, "could not create bucket") - } - data, err := json.Marshal(visitor) - if err != nil { - return errors.Wrap(err, "could not create json") - } - return bucket.Put([]byte(requestID), data) - }) - if err != nil { - logrus.Warningf("could not register visit: %v", err) - } -} - -// GetVisitors returns all the visits of a shorted URL -func (s *Store) GetVisitors(id string) ([]Visitor, error) { - output := []Visitor{} - return output, s.db.Update(func(tx *bolt.Tx) error { - bucket, err := tx.CreateBucketIfNotExists([]byte(id)) - if err != nil { - return errors.Wrap(err, "could not create bucket") - } - return bucket.ForEach(func(k, v []byte) error { - var value Visitor - if err := json.Unmarshal(v, &value); err != nil { - return errors.Wrap(err, "could not unmarshal json") - } - output = append(output, value) - return nil - }) - }) -} - -// GetUserEntries returns all the shorted URL entries of an user -func (s *Store) GetUserEntries(oAuthProvider, oAuthID string) (map[string]Entry, error) { - entries := map[string]Entry{} - return entries, s.db.Update(func(tx *bolt.Tx) error { - bucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) - if err != nil { - return errors.Wrap(err, "could not create bucket") - } - return bucket.ForEach(func(k, v []byte) error { - if bytes.Equal(v, []byte(oAuthProvider+oAuthID)) { - entry, err := s.GetEntryByID(string(k)) - if err != nil { - return errors.Wrap(err, "could not get entry") - } - entries[string(k)] = *entry - } - return nil - }) - }) -} - -// Close closes the bolt db database -func (s *Store) Close() error { - return s.db.Close() -} diff --git a/store/util.go b/store/util.go deleted file mode 100644 index 3087c10..0000000 --- a/store/util.go +++ /dev/null @@ -1,70 +0,0 @@ -package store - -import ( - "crypto/hmac" - "crypto/rand" - "crypto/sha512" - "encoding/json" - "math/big" - "time" - "unicode" - - "github.com/boltdb/bolt" - "github.com/maxibanki/golang-url-shortener/util" - "github.com/pkg/errors" -) - -// createEntryRaw creates a entry with the given key value pair -func (s *Store) createEntryRaw(key, value, userIdentifier []byte) error { - return s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(shortedURLsBucket) - if raw := bucket.Get(key); raw != nil { - return errors.New("entry already exists") - } - if err := bucket.Put(key, value); err != nil { - return errors.Wrap(err, "could not put data into bucket") - } - uTsURLsBucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) - if err != nil { - return errors.Wrap(err, "could not create bucket") - } - return uTsURLsBucket.Put(key, userIdentifier) - }) -} - -// createEntry creates a new entry with a randomly generated id. If on is present -// then the given ID is used -func (s *Store) createEntry(entry Entry, entryID string) (string, []byte, error) { - var err error - if entryID == "" { - if entryID, err = generateRandomString(s.idLength); err != nil { - return "", nil, errors.Wrap(err, "could not generate random string") - } - } - entry.Public.CreatedOn = time.Now() - rawEntry, err := json.Marshal(entry) - if err != nil { - return "", nil, err - } - mac := hmac.New(sha512.New, util.GetPrivateKey()) - if _, err := mac.Write([]byte(entryID)); err != nil { - return "", nil, errors.Wrap(err, "could not write hmac") - } - return entryID, mac.Sum(nil), s.createEntryRaw([]byte(entryID), rawEntry, []byte(entry.OAuthProvider+entry.OAuthID)) -} - -// generateRandomString generates a random string with an predefined length -func generateRandomString(length int) (string, error) { - var result string - for len(result) < length { - num, err := rand.Int(rand.Reader, big.NewInt(int64(127))) - if err != nil { - return "", err - } - n := num.Int64() - if unicode.IsLetter(rune(n)) { - result += string(n) - } - } - return result, nil -} diff --git a/stores/boltdb/bolt.go b/stores/boltdb/bolt.go new file mode 100644 index 0000000..5f33b24 --- /dev/null +++ b/stores/boltdb/bolt.go @@ -0,0 +1,193 @@ +package boltdb + +import ( + "bytes" + "encoding/json" + "time" + + "github.com/boltdb/bolt" + "github.com/maxibanki/golang-url-shortener/stores/shared" + "github.com/pkg/errors" +) + +var ( + shortedURLsBucket = []byte("shorted") + shortedIDsToUserBucket = []byte("shorted2Users") +) + +// BoltStore implements the stores.Storage interface +type BoltStore struct { + db *bolt.DB +} + +// New returns a bolt store which implements the stores.Storage interface +func New(path string) (*BoltStore, error) { + db, err := bolt.Open(path, 0644, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return nil, errors.Wrap(err, "could not open bolt DB database") + } + err = db.Update(func(tx *bolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists(shortedURLsBucket); err != nil { + return errors.Wrapf(err, "could not create %s bucket", shortedURLsBucket) + } + if _, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket); err != nil { + return errors.Wrapf(err, "could not create %s bucket", shortedIDsToUserBucket) + } + return err + }) + if err != nil { + return nil, errors.Wrap(err, "could not create buckets") + } + return &BoltStore{ + db: db, + }, nil +} + +// Close closes the bolt database +func (b *BoltStore) Close() error { + return b.db.Close() +} + +// GetEntryByID returns a entry and an error by the shorted ID +func (b *BoltStore) GetEntryByID(id string) (*shared.Entry, error) { + var raw []byte + err := b.db.View(func(tx *bolt.Tx) error { + raw = tx.Bucket(shortedURLsBucket).Get([]byte(id)) + if raw == nil { + return shared.ErrNoEntryFound + } + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "could not view db") + } + var entry *shared.Entry + return entry, json.Unmarshal(raw, &entry) +} + +// IncreaseVisitCounter increases the visit counter and sets the current +// time as the last visit ones +func (b *BoltStore) IncreaseVisitCounter(id string) error { + entry, err := b.GetEntryByID(id) + if err != nil { + return errors.Wrap(err, "could not get entry by ID") + } + entry.Public.VisitCount++ + currentTime := time.Now() + entry.Public.LastVisit = ¤tTime + raw, err := json.Marshal(entry) + if err != nil { + return errors.Wrap(err, "could not marshal json") + } + err = b.db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket(shortedURLsBucket).Put([]byte(id), raw); err != nil { + return errors.Wrap(err, "could not put updated visitor") + } + return nil + }) + return errors.Wrap(err, "could not update entry") +} + +// CreateEntry creates an entry by a given ID and returns an error +func (b *BoltStore) CreateEntry(entry shared.Entry, id, userIdentifier string) error { + entryRaw, err := json.Marshal(entry) + if err != nil { + return errors.Wrap(err, "could not marshal entry") + } + err = b.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(shortedURLsBucket) + if raw := bucket.Get([]byte(id)); raw != nil { + return errors.New("entry already exists") + } + if err := bucket.Put([]byte(id), entryRaw); err != nil { + return errors.Wrap(err, "could not put data into bucket") + } + uTsURLsBucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return uTsURLsBucket.Put([]byte(id), []byte(userIdentifier)) + }) + return errors.Wrap(err, "could not update db") +} + +// DeleteEntry deleted an entry by a given ID and returns an error +func (b *BoltStore) DeleteEntry(id string) error { + err := b.db.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(shortedURLsBucket) + if bucket.Get([]byte(id)) == nil { + return errors.New("entry already deleted") + } + if err := bucket.Delete([]byte(id)); err != nil { + return errors.Wrap(err, "could not delete entry") + } + if err := tx.DeleteBucket([]byte(id)); err != nil && err != bolt.ErrBucketNotFound && err != bolt.ErrBucketExists { + return errors.Wrap(err, "could not delte bucket") + } + uTsIDsBucket := tx.Bucket(shortedIDsToUserBucket) + return uTsIDsBucket.ForEach(func(k, v []byte) error { + if bytes.Equal(k, []byte(id)) { + return uTsIDsBucket.Delete(k) + } + return nil + }) + }) + return errors.Wrap(err, "could not update db") +} + +// GetVisitors returns the visitors and an error of an entry +func (b *BoltStore) GetVisitors(id string) ([]shared.Visitor, error) { + output := []shared.Visitor{} + return output, b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(id)) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return bucket.ForEach(func(k, v []byte) error { + var value shared.Visitor + if err := json.Unmarshal(v, &value); err != nil { + return errors.Wrap(err, "could not unmarshal json") + } + output = append(output, value) + return nil + }) + }) +} + +// GetUserEntries returns all user entries of an given user identifier +func (b *BoltStore) GetUserEntries(userIdentifier string) (map[string]shared.Entry, error) { + entries := map[string]shared.Entry{} + err := b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return bucket.ForEach(func(k, v []byte) error { + if bytes.Equal(v, []byte(userIdentifier)) { + entry, err := b.GetEntryByID(string(k)) + if err != nil { + return errors.Wrap(err, "could not get entry") + } + entries[string(k)] = *entry + } + return nil + }) + }) + return entries, errors.Wrap(err, "could not update db") +} + +// RegisterVisitor saves the visitor in the database +func (b *BoltStore) RegisterVisitor(id, visitID string, visitor shared.Visitor) error { + err := b.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(id)) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + data, err := json.Marshal(visitor) + if err != nil { + return errors.Wrap(err, "could not create json") + } + return bucket.Put([]byte(visitID), data) + }) + return errors.Wrap(err, "could not update db") +} diff --git a/stores/shared/shared.go b/stores/shared/shared.go new file mode 100644 index 0000000..eca8868 --- /dev/null +++ b/stores/shared/shared.go @@ -0,0 +1,46 @@ +package shared + +import ( + "errors" + "time" +) + +// Storage is an interface which will be implmented by each storage +// e.g. bolt, sqlite +type Storage interface { + GetEntryByID(string) (*Entry, error) + GetVisitors(string) ([]Visitor, error) + DeleteEntry(string) error + IncreaseVisitCounter(string) error + CreateEntry(Entry, string, string) error + GetUserEntries(string) (map[string]Entry, error) + RegisterVisitor(string, string, Visitor) error + Close() error +} + +// Entry is the data set which is stored in the DB as JSON +type Entry struct { + OAuthProvider, OAuthID string + RemoteAddr string `json:",omitempty"` + DeletionURL string `json:",omitempty"` + Password []byte `json:",omitempty"` + Public EntryPublicData +} + +// EntryPublicData is the public part of an entry +type EntryPublicData struct { + CreatedOn time.Time + LastVisit, Expiration *time.Time `json:",omitempty"` + VisitCount int + URL string +} + +// Visitor is the entry which is stored in the visitors bucket +type Visitor struct { + IP, Referer, UserAgent string + Timestamp time.Time + UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm string `json:",omitempty"` +} + +// ErrNoEntryFound is returned when no entry to a id is found +var ErrNoEntryFound = errors.New("no entry found with this ID") diff --git a/stores/sqlite/sqlite.go b/stores/sqlite/sqlite.go new file mode 100644 index 0000000..fef43c1 --- /dev/null +++ b/stores/sqlite/sqlite.go @@ -0,0 +1 @@ +package sqlite diff --git a/stores/store.go b/stores/store.go new file mode 100644 index 0000000..e790ddb --- /dev/null +++ b/stores/store.go @@ -0,0 +1,189 @@ +// Package stores provides support to interact with the entries +package stores + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha512" + "math/big" + "path/filepath" + "strings" + "time" + "unicode" + + "github.com/asaskevich/govalidator" + "github.com/maxibanki/golang-url-shortener/stores/boltdb" + "github.com/maxibanki/golang-url-shortener/stores/shared" + "github.com/maxibanki/golang-url-shortener/util" + "github.com/pborman/uuid" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/bcrypt" +) + +// Store holds internal funcs and vars about the store +type Store struct { + storage shared.Storage + idLength int +} + +// ErrNoValidURL is returned when the URL is not valid +var ErrNoValidURL = errors.New("the given URL is no valid URL") + +// ErrGeneratingIDFailed is returned when the 10 tries to generate an id failed +var ErrGeneratingIDFailed = errors.New("could not generate unique id, all ten tries failed") + +// ErrEntryIsExpired is returned when the entry is expired +var ErrEntryIsExpired = errors.New("entry is expired") + +// New initializes the store with the db +func New() (*Store, error) { + s, err := boltdb.New(filepath.Join(util.GetConfig().DataDir, "main.db")) + if err != nil { + return nil, errors.Wrap(err, "could not create bolt db store") + } + return &Store{ + storage: s, + idLength: util.GetConfig().ShortedIDLength, + }, nil +} + +// GetEntryByID returns a unmarshalled entry of the db by a given ID +func (s *Store) GetEntryByID(id string) (*shared.Entry, error) { + if id == "" { + return nil, shared.ErrNoEntryFound + } + return s.storage.GetEntryByID(id) +} + +// GetEntryAndIncrease Increases the visitor count, checks +// if the URL is expired and returns the origin URL +func (s *Store) GetEntryAndIncrease(id string) (*shared.Entry, error) { + entry, err := s.GetEntryByID(id) + if err != nil { + return nil, err + } + if entry.Public.Expiration != nil && time.Now().After(*entry.Public.Expiration) { + return nil, ErrEntryIsExpired + } + if err := s.storage.IncreaseVisitCounter(id); err != nil { + return nil, errors.Wrap(err, "could not increase visitor counter") + } + return entry, nil +} + +// CreateEntry creates a new record and returns his short id +func (s *Store) CreateEntry(entry shared.Entry, givenID, password string) (string, []byte, error) { + entry.Public.URL = strings.Replace(entry.Public.URL, " ", "%20", -1) + if !govalidator.IsURL(entry.Public.URL) { + return "", nil, ErrNoValidURL + } + if password != "" { + var err error + entry.Password, err = bcrypt.GenerateFromPassword([]byte(password), 10) + if err != nil { + return "", nil, errors.Wrap(err, "could not generate bcrypt from password") + } + } + // try it 10 times to make a short URL + for i := 1; i <= 10; i++ { + id, delID, err := s.createEntry(entry, givenID) + if err != nil && givenID != "" { + return "", nil, err + } else if err != nil { + logrus.Debugf("Could not create entry: %v", err) + continue + } + return id, delID, nil + } + return "", nil, ErrGeneratingIDFailed +} + +// DeleteEntry deletes an Entry fully from the DB +func (s *Store) DeleteEntry(id string, givenHmac []byte) error { + mac := hmac.New(sha512.New, util.GetPrivateKey()) + if _, err := mac.Write([]byte(id)); err != nil { + return errors.Wrap(err, "could not write hmac") + } + if !hmac.Equal(mac.Sum(nil), givenHmac) { + return errors.New("hmac verification failed") + } + return errors.Wrap(s.storage.DeleteEntry(id), "could not delete entry") +} + +// RegisterVisit registers an new incoming request in the store +func (s *Store) RegisterVisit(id string, visitor shared.Visitor) { + requestID := uuid.New() + logrus.WithFields(logrus.Fields{ + "ClientIP": visitor.IP, + "ID": id, + "RequestID": requestID, + }).Info("New redirect was registered...") + if err := s.storage.RegisterVisitor(id, requestID, visitor); err != nil { + logrus.Warningf("could not register visit: %v", err) + } +} + +// GetVisitors returns all the visits of a shorted URL +func (s *Store) GetVisitors(id string) ([]shared.Visitor, error) { + visitors, err := s.storage.GetVisitors(id) + if err != nil { + return nil, errors.Wrap(err, "could not get visitors") + } + return visitors, nil +} + +// GetUserEntries returns all the shorted URL entries of an user +func (s *Store) GetUserEntries(oAuthProvider, oAuthID string) (map[string]shared.Entry, error) { + userIdentifier := getUserIdentifier(oAuthProvider, oAuthID) + entries, err := s.storage.GetUserEntries(userIdentifier) + if err != nil { + return nil, errors.Wrap(err, "could not get user entries") + } + return entries, nil +} + +func getUserIdentifier(oAuthProvider, oAuthID string) string { + return oAuthProvider + oAuthID +} + +// Close closes the bolt db database +func (s *Store) Close() error { + return s.storage.Close() +} + +// createEntry creates a new entry with a randomly generated id. If on is present +// then the given ID is used +func (s *Store) createEntry(entry shared.Entry, entryID string) (string, []byte, error) { + var err error + if entryID == "" { + if entryID, err = generateRandomString(s.idLength); err != nil { + return "", nil, errors.Wrap(err, "could not generate random string") + } + } + entry.Public.CreatedOn = time.Now() + mac := hmac.New(sha512.New, util.GetPrivateKey()) + if _, err := mac.Write([]byte(entryID)); err != nil { + return "", nil, errors.Wrap(err, "could not write hmac") + } + if err := s.storage.CreateEntry(entry, entryID, getUserIdentifier(entry.OAuthProvider, entry.OAuthID)); err != nil { + return "", nil, errors.Wrap(err, "could not create entry") + } + return entryID, mac.Sum(nil), nil +} + +// generateRandomString generates a random string with an predefined length +func generateRandomString(length int) (string, error) { + var result string + for len(result) < length { + num, err := rand.Int(rand.Reader, big.NewInt(int64(127))) + if err != nil { + return "", err + } + n := num.Int64() + if unicode.IsLetter(rune(n)) { + result += string(n) + } + } + return result, nil +} diff --git a/store/store_test.go b/stores/store_test.go similarity index 65% rename from store/store_test.go rename to stores/store_test.go index 0329edf..2183e57 100644 --- a/store/store_test.go +++ b/stores/store_test.go @@ -1,9 +1,10 @@ -package store +package stores import ( "strings" "testing" + "github.com/maxibanki/golang-url-shortener/stores/shared" "github.com/maxibanki/golang-url-shortener/util" ) @@ -55,13 +56,13 @@ func TestCreateEntry(t *testing.T) { t.Fatalf("unexpected error: %v", err) } defer cleanup(store) - _, _, err = store.CreateEntry(Entry{}, "", "") + _, _, err = store.CreateEntry(shared.Entry{}, "", "") if err != ErrNoValidURL { t.Fatalf("unexpected error: %v", err) } for i := 1; i <= 100; i++ { - _, _, err := store.CreateEntry(Entry{ - Public: EntryPublicData{ + _, _, err := store.CreateEntry(shared.Entry{ + Public: shared.EntryPublicData{ URL: "https://golang.org/", }, }, "", "") @@ -78,45 +79,12 @@ func TestGetEntryByID(t *testing.T) { } defer cleanup(store) _, err = store.GetEntryByID("something that not exists") - if err != ErrNoEntryFound { - t.Fatalf("could not get expected '%v' error: %v", ErrNoEntryFound, err) + if !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) { + t.Fatalf("could not get expected '%v' error: %v", shared.ErrNoEntryFound, err) } _, err = store.GetEntryByID("") - if err != ErrNoEntryFound { - t.Fatalf("could not get expected '%v' error: %v", ErrNoEntryFound, err) - } -} - -func TestIncreaseVisitCounter(t *testing.T) { - store, err := New() - if err != nil { - t.Fatalf("could not create store: %v", err) - } - defer cleanup(store) - id, _, err := store.CreateEntry(Entry{ - Public: EntryPublicData{ - URL: "https://golang.org/", - }, - }, "", "") - if err != nil { - t.Fatalf("could not create entry: %v", err) - } - entryBeforeInc, err := store.GetEntryByID(id) - if err != nil { - t.Fatalf("could not get entry by id: %v", err) - } - if err = store.IncreaseVisitCounter(id); err != nil { - t.Fatalf("could not increase visit counter %v", err) - } - entryAfterInc, err := store.GetEntryByID(id) - if err != nil { - t.Fatalf("could not get entry by id: %v", err) - } - if entryBeforeInc.Public.VisitCount+1 != entryAfterInc.Public.VisitCount { - t.Fatalf("the increasement was not successful, the visit count is not correct") - } - if err = store.IncreaseVisitCounter(""); !strings.Contains(err.Error(), ErrNoEntryFound.Error()) { - t.Fatalf("could not get expected '%v'; got: %v", ErrNoEntryFound, err) + if !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) { + t.Fatalf("could not get expected '%v' error: %v", shared.ErrNoEntryFound, err) } } @@ -126,8 +94,8 @@ func TestDelete(t *testing.T) { t.Fatalf("could not create store: %v", err) } defer cleanup(store) - entryID, delHMac, err := store.CreateEntry(Entry{ - Public: EntryPublicData{ + entryID, delHMac, err := store.CreateEntry(shared.Entry{ + Public: shared.EntryPublicData{ URL: "https://golang.org/", }, }, "", "") @@ -137,7 +105,7 @@ func TestDelete(t *testing.T) { if err := store.DeleteEntry(entryID, delHMac); err != nil { t.Fatalf("could not delete entry: %v", err) } - if _, err := store.GetEntryByID(entryID); err != ErrNoEntryFound { + if _, err := store.GetEntryByID(entryID); !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) { t.Fatalf("unexpected error: %v", err) } } @@ -149,8 +117,8 @@ func TestGetURLAndIncrease(t *testing.T) { } defer cleanup(store) const url = "https://golang.org/" - entryID, _, err := store.CreateEntry(Entry{ - Public: EntryPublicData{ + entryID, _, err := store.CreateEntry(shared.Entry{ + Public: shared.EntryPublicData{ URL: url, }, }, "", "") diff --git a/util/config.go b/util/config.go index fd51c99..4fddeb6 100644 --- a/util/config.go +++ b/util/config.go @@ -51,7 +51,7 @@ func ReadInConfig() error { } else if !os.IsNotExist(err) { return errors.Wrap(err, "could not read config file") } else { - logrus.Info("No configuration file found, using defaults and environment variable overrides.") + logrus.Info("No configuration file found, using defaults with environment variable overrides.") } if err := config.ApplyEnvironmentConfig(); err != nil { return errors.Wrap(err, "could not apply environment configuration")