nicolas
/
golang-url-shortener
mirror of https://github.com/nmasse-itix/golang-url-shortener
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Abstracted boltdb as a storage interface (fix #60)

dependabot/npm_and_yarn/web/prismjs-1.21.0
Max Schmitt 8 years ago
parent
6a5574d882
commit
071c94b2d7
13 changed files with 472 additions and 417 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      handlers/auth_test.go
  2. 6
      handlers/handlers.go
  3. 24
      handlers/public.go
  4. 9
      handlers/public_test.go
  5. 9
      main.go
  6. 276
      store/store.go
  7. 70
      store/util.go
  8. 193
      stores/boltdb/bolt.go
  9. 46
      stores/shared/shared.go
  10. 1
      stores/sqlite/sqlite.go
  11. 189
      stores/store.go
  12. 60
      stores/store_test.go
  13. 2
      util/config.go

4
handlers/auth_test.go
View File

@ -12,7 +12,7 @@ import (
jwt "github.com/dgrijalva/jwt-go" jwt "github.com/dgrijalva/jwt-go"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/maxibanki/golang-url-shortener/handlers/auth" "github.com/maxibanki/golang-url-shortener/handlers/auth"
"github.com/maxibanki/golang-url-shortener/store" "github.com/maxibanki/golang-url-shortener/stores"
"github.com/maxibanki/golang-url-shortener/util" "github.com/maxibanki/golang-url-shortener/util"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -38,7 +38,7 @@ func TestCreateBackend(t *testing.T) {
if err := util.ReadInConfig(); err != nil { if err := util.ReadInConfig(); err != nil {
t.Fatalf("could not reload config file: %v", err) t.Fatalf("could not reload config file: %v", err)
} }
store, err := store.New() store, err := stores.New()
if err != nil { if err != nil {
t.Fatalf("could not create store: %v", err) t.Fatalf("could not create store: %v", err)
} }

6
handlers/handlers.go
View File

@ -11,7 +11,7 @@ import (
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/maxibanki/golang-url-shortener/handlers/tmpls" "github.com/maxibanki/golang-url-shortener/handlers/tmpls"
"github.com/maxibanki/golang-url-shortener/store" "github.com/maxibanki/golang-url-shortener/stores"
"github.com/maxibanki/golang-url-shortener/util" "github.com/maxibanki/golang-url-shortener/util"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -19,7 +19,7 @@ import (
// Handler holds the funcs and attributes for the // Handler holds the funcs and attributes for the
// http communication // http communication
type Handler struct { type Handler struct {
store store.Store store stores.Store
engine *gin.Engine engine *gin.Engine
providers []string providers []string
} }
@ -28,7 +28,7 @@ type Handler struct {
var DoNotPrivateKeyChecking = false var DoNotPrivateKeyChecking = false
// New initializes the http handlers // New initializes the http handlers
func New(store store.Store) (*Handler, error) { func New(store stores.Store) (*Handler, error) {
if !util.GetConfig().EnableDebugMode { if !util.GetConfig().EnableDebugMode {
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
} }

24
handlers/public.go
View File

@ -8,11 +8,12 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"runtime" "runtime"
"strings"
"time" "time"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/maxibanki/golang-url-shortener/handlers/auth" "github.com/maxibanki/golang-url-shortener/handlers/auth"
"github.com/maxibanki/golang-url-shortener/store" "github.com/maxibanki/golang-url-shortener/stores/shared"
"github.com/maxibanki/golang-url-shortener/util" "github.com/maxibanki/golang-url-shortener/util"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
) )
@ -40,8 +41,8 @@ func (h *Handler) handleLookup(c *gin.Context) {
return return
} }
if !h.oAuthPropertiesEquals(c, entry.OAuthID, entry.OAuthProvider) { if !h.oAuthPropertiesEquals(c, entry.OAuthID, entry.OAuthProvider) {
c.JSON(http.StatusOK, store.Entry{ c.JSON(http.StatusOK, shared.Entry{
Public: store.EntryPublicData{ Public: shared.EntryPublicData{
URL: entry.Public.URL, URL: entry.Public.URL,
}, },
}) })
@ -54,11 +55,12 @@ func (h *Handler) handleLookup(c *gin.Context) {
func (h *Handler) handleAccess(c *gin.Context) { func (h *Handler) handleAccess(c *gin.Context) {
id := c.Request.URL.Path[1:] id := c.Request.URL.Path[1:]
entry, err := h.store.GetEntryAndIncrease(id) entry, err := h.store.GetEntryAndIncrease(id)
if err == store.ErrNoEntryFound { if err != nil {
return if strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) {
} else if err != nil { return
http.Error(c.Writer, fmt.Sprintf("could not get and crease visitor counter: %v, ", err), http.StatusInternalServerError) http.Error(c.Writer, fmt.Sprintf("could not get and crease visitor counter: %v, ", err), http.StatusInternalServerError)
return return
}
} }
// No password set // No password set
if len(entry.Password) == 0 { if len(entry.Password) == 0 {
@ -100,8 +102,8 @@ func (h *Handler) handleCreate(c *gin.Context) {
return return
} }
user := c.MustGet("user").(*auth.JWTClaims) user := c.MustGet("user").(*auth.JWTClaims)
id, delID, err := h.store.CreateEntry(store.Entry{ id, delID, err := h.store.CreateEntry(shared.Entry{
Public: store.EntryPublicData{ Public: shared.EntryPublicData{
URL: data.URL, URL: data.URL,
Expiration: data.Expiration, Expiration: data.Expiration,
}, },
@ -190,7 +192,7 @@ func (h *Handler) getURLOrigin(c *gin.Context) string {
} }
func (h *Handler) registerVisitor(id string, c *gin.Context) { func (h *Handler) registerVisitor(id string, c *gin.Context) {
h.store.RegisterVisit(id, store.Visitor{ h.store.RegisterVisit(id, shared.Visitor{
IP: c.ClientIP(), IP: c.ClientIP(),
Timestamp: time.Now(), Timestamp: time.Now(),
Referer: c.GetHeader("Referer"), Referer: c.GetHeader("Referer"),

9
handlers/public_test.go
View File

@ -3,6 +3,7 @@ package handlers
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
@ -10,7 +11,8 @@ import (
"testing" "testing"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/maxibanki/golang-url-shortener/store" "github.com/maxibanki/golang-url-shortener/stores"
"github.com/maxibanki/golang-url-shortener/stores/shared"
) )
const testURL = "https://www.google.de/" const testURL = "https://www.google.de/"
@ -50,7 +52,7 @@ func TestCreateEntry(t *testing.T) {
}, },
statusCode: http.StatusBadRequest, statusCode: http.StatusBadRequest,
contentType: "application/json; charset=utf-8", contentType: "application/json; charset=utf-8",
response: gin.H{"error": store.ErrNoValidURL.Error()}, response: gin.H{"error": stores.ErrNoValidURL.Error()},
ignoreResponse: true, ignoreResponse: true,
}, },
} }
@ -121,7 +123,7 @@ func TestHandleInfo(t *testing.T) {
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
t.Errorf("expected status %d; got %d", http.StatusOK, resp.StatusCode) t.Errorf("expected status %d; got %d", http.StatusOK, resp.StatusCode)
} }
var entry store.EntryPublicData var entry shared.EntryPublicData
if err = json.NewDecoder(resp.Body).Decode(&entry); err != nil { if err = json.NewDecoder(resp.Body).Decode(&entry); err != nil {
t.Fatalf("could not unmarshal data: %v", err) t.Fatalf("could not unmarshal data: %v", err)
} }
@ -272,6 +274,7 @@ func TestHandleDeletion(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("could not send visit request: %v", err) t.Fatalf("could not send visit request: %v", err)
} }
fmt.Println(body.URL)
if resp.StatusCode != http.StatusNotFound { if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected status: %d; got: %d", http.StatusNotFound, resp.StatusCode) t.Fatalf("expected status: %d; got: %d", http.StatusNotFound, resp.StatusCode)
} }

9
main.go
View File

@ -4,13 +4,12 @@ import (
"os" "os"
"os/signal" "os/signal"
"github.com/shiena/ansicolor"
"github.com/sirupsen/logrus"
"github.com/maxibanki/golang-url-shortener/handlers" "github.com/maxibanki/golang-url-shortener/handlers"
"github.com/maxibanki/golang-url-shortener/store" "github.com/maxibanki/golang-url-shortener/stores"
"github.com/maxibanki/golang-url-shortener/util" "github.com/maxibanki/golang-url-shortener/util"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/shiena/ansicolor"
"github.com/sirupsen/logrus"
) )
func main() { func main() {
@ -36,7 +35,7 @@ func initShortener() (func(), error) {
if util.GetConfig().EnableDebugMode { if util.GetConfig().EnableDebugMode {
logrus.SetLevel(logrus.DebugLevel) logrus.SetLevel(logrus.DebugLevel)
} }
store, err := store.New() store, err := stores.New()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "could not create store") return nil, errors.Wrap(err, "could not create store")
} }

276
store/store.go
View File

@ -1,276 +0,0 @@
// Package store provides support to interact with the entries
package store
import (
"bytes"
"crypto/hmac"
"crypto/sha512"
"encoding/json"
"path/filepath"
"strings"
"time"
"github.com/maxibanki/golang-url-shortener/util"
"github.com/pborman/uuid"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
"github.com/asaskevich/govalidator"
"github.com/boltdb/bolt"
"github.com/pkg/errors"
)
// Store holds internal funcs and vars about the store
type Store struct {
db *bolt.DB
idLength int
}
// Entry is the data set which is stored in the DB as JSON
type Entry struct {
OAuthProvider, OAuthID string
RemoteAddr string `json:",omitempty"`
DeletionURL string `json:",omitempty"`
Password []byte `json:",omitempty"`
Public EntryPublicData
}
// Visitor is the entry which is stored in the visitors bucket
type Visitor struct {
IP, Referer, UserAgent string
Timestamp time.Time
UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm string `json:",omitempty"`
}
// EntryPublicData is the public part of an entry
type EntryPublicData struct {
CreatedOn time.Time
LastVisit, Expiration *time.Time `json:",omitempty"`
VisitCount int
URL string
}
// ErrNoEntryFound is returned when no entry to a id is found
var ErrNoEntryFound = errors.New("no entry found with this ID")
// ErrNoValidURL is returned when the URL is not valid
var ErrNoValidURL = errors.New("the given URL is no valid URL")
// ErrGeneratingIDFailed is returned when the 10 tries to generate an id failed
var ErrGeneratingIDFailed = errors.New("could not generate unique id, all ten tries failed")
// ErrEntryIsExpired is returned when the entry is expired
var ErrEntryIsExpired = errors.New("entry is expired")
var (
shortedURLsBucket = []byte("shorted")
shortedIDsToUserBucket = []byte("shorted2Users")
)
// New initializes the store with the db
func New() (*Store, error) {
db, err := bolt.Open(filepath.Join(util.GetConfig().DataDir, "main.db"), 0644, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, errors.Wrap(err, "could not open bolt DB database")
}
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(shortedURLsBucket)
return err
})
if err != nil {
return nil, err
}
return &Store{
db: db,
idLength: util.GetConfig().ShortedIDLength,
}, nil
}
// GetEntryByID returns a unmarshalled entry of the db by a given ID
func (s *Store) GetEntryByID(id string) (*Entry, error) {
if id == "" {
return nil, ErrNoEntryFound
}
rawEntry, err := s.GetEntryByIDRaw(id)
if err != nil {
return nil, err
}
var entry *Entry
return entry, json.Unmarshal(rawEntry, &entry)
}
// IncreaseVisitCounter increments the visit counter of an entry
func (s *Store) IncreaseVisitCounter(id string) error {
entry, err := s.GetEntryByID(id)
if err != nil {
return errors.Wrap(err, "could not get entry by ID")
}
entry.Public.VisitCount++
currentTime := time.Now()
entry.Public.LastVisit = &currentTime
raw, err := json.Marshal(entry)
if err != nil {
return err
}
return s.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(shortedURLsBucket).Put([]byte(id), raw); err != nil {
return errors.Wrap(err, "could not put updated visitor count JSON into the bucket")
}
return nil
})
}
// GetEntryAndIncrease Increases the visitor count, checks
// if the URL is expired and returns the origin URL
func (s *Store) GetEntryAndIncrease(id string) (*Entry, error) {
entry, err := s.GetEntryByID(id)
if err != nil {
return nil, err
}
if entry.Public.Expiration != nil && time.Now().After(*entry.Public.Expiration) {
return nil, ErrEntryIsExpired
}
if err := s.IncreaseVisitCounter(id); err != nil {
return nil, errors.Wrap(err, "could not increase visitor counter")
}
return entry, nil
}
// GetEntryByIDRaw returns the raw data (JSON) of a data set
func (s *Store) GetEntryByIDRaw(id string) ([]byte, error) {
var raw []byte
return raw, s.db.View(func(tx *bolt.Tx) error {
raw = tx.Bucket(shortedURLsBucket).Get([]byte(id))
if raw == nil {
return ErrNoEntryFound
}
return nil
})
}
// CreateEntry creates a new record and returns his short id
func (s *Store) CreateEntry(entry Entry, givenID, password string) (string, []byte, error) {
entry.Public.URL = strings.Replace(entry.Public.URL, " ", "%20", -1)
if !govalidator.IsURL(entry.Public.URL) {
return "", nil, ErrNoValidURL
}
if password != "" {
var err error
entry.Password, err = bcrypt.GenerateFromPassword([]byte(password), 10)
if err != nil {
return "", nil, errors.Wrap(err, "could not generate bcrypt from password")
}
}
// try it 10 times to make a short URL
for i := 1; i <= 10; i++ {
id, delID, err := s.createEntry(entry, givenID)
if err != nil && givenID != "" {
return "", nil, err
} else if err != nil {
logrus.Debugf("Could not create entry: %v", err)
continue
}
return id, delID, nil
}
return "", nil, ErrGeneratingIDFailed
}
// DeleteEntry deletes an Entry fully from the DB
func (s *Store) DeleteEntry(id string, givenHmac []byte) error {
mac := hmac.New(sha512.New, util.GetPrivateKey())
if _, err := mac.Write([]byte(id)); err != nil {
return errors.Wrap(err, "could not write hmac")
}
if !hmac.Equal(mac.Sum(nil), givenHmac) {
return errors.New("hmac verification failed")
}
return s.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(shortedURLsBucket)
if bucket.Get([]byte(id)) == nil {
return errors.New("entry already deleted")
}
if err := bucket.Delete([]byte(id)); err != nil {
return errors.Wrap(err, "could not delete entry")
}
if err := tx.DeleteBucket([]byte(id)); err != nil && err != bolt.ErrBucketNotFound && err != bolt.ErrBucketExists {
return errors.Wrap(err, "could not delte bucket")
}
uTsIDsBucket := tx.Bucket(shortedIDsToUserBucket)
return uTsIDsBucket.ForEach(func(k, v []byte) error {
if bytes.Equal(k, []byte(id)) {
return uTsIDsBucket.Delete(k)
}
return nil
})
})
}
// RegisterVisit registers an new incoming request in the store
func (s *Store) RegisterVisit(id string, visitor Visitor) {
requestID := uuid.New()
logrus.WithFields(logrus.Fields{
"ClientIP": visitor.IP,
"ID": id,
"RequestID": requestID,
}).Info("New redirect was registered...")
err := s.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(id))
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
data, err := json.Marshal(visitor)
if err != nil {
return errors.Wrap(err, "could not create json")
}
return bucket.Put([]byte(requestID), data)
})
if err != nil {
logrus.Warningf("could not register visit: %v", err)
}
}
// GetVisitors returns all the visits of a shorted URL
func (s *Store) GetVisitors(id string) ([]Visitor, error) {
output := []Visitor{}
return output, s.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(id))
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return bucket.ForEach(func(k, v []byte) error {
var value Visitor
if err := json.Unmarshal(v, &value); err != nil {
return errors.Wrap(err, "could not unmarshal json")
}
output = append(output, value)
return nil
})
})
}
// GetUserEntries returns all the shorted URL entries of an user
func (s *Store) GetUserEntries(oAuthProvider, oAuthID string) (map[string]Entry, error) {
entries := map[string]Entry{}
return entries, s.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket)
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return bucket.ForEach(func(k, v []byte) error {
if bytes.Equal(v, []byte(oAuthProvider+oAuthID)) {
entry, err := s.GetEntryByID(string(k))
if err != nil {
return errors.Wrap(err, "could not get entry")
}
entries[string(k)] = *entry
}
return nil
})
})
}
// Close closes the bolt db database
func (s *Store) Close() error {
return s.db.Close()
}

70
store/util.go
View File

@ -1,70 +0,0 @@
package store
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha512"
"encoding/json"
"math/big"
"time"
"unicode"
"github.com/boltdb/bolt"
"github.com/maxibanki/golang-url-shortener/util"
"github.com/pkg/errors"
)
// createEntryRaw creates a entry with the given key value pair
func (s *Store) createEntryRaw(key, value, userIdentifier []byte) error {
return s.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(shortedURLsBucket)
if raw := bucket.Get(key); raw != nil {
return errors.New("entry already exists")
}
if err := bucket.Put(key, value); err != nil {
return errors.Wrap(err, "could not put data into bucket")
}
uTsURLsBucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket)
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return uTsURLsBucket.Put(key, userIdentifier)
})
}
// createEntry creates a new entry with a randomly generated id. If on is present
// then the given ID is used
func (s *Store) createEntry(entry Entry, entryID string) (string, []byte, error) {
var err error
if entryID == "" {
if entryID, err = generateRandomString(s.idLength); err != nil {
return "", nil, errors.Wrap(err, "could not generate random string")
}
}
entry.Public.CreatedOn = time.Now()
rawEntry, err := json.Marshal(entry)
if err != nil {
return "", nil, err
}
mac := hmac.New(sha512.New, util.GetPrivateKey())
if _, err := mac.Write([]byte(entryID)); err != nil {
return "", nil, errors.Wrap(err, "could not write hmac")
}
return entryID, mac.Sum(nil), s.createEntryRaw([]byte(entryID), rawEntry, []byte(entry.OAuthProvider+entry.OAuthID))
}
// generateRandomString generates a random string with an predefined length
func generateRandomString(length int) (string, error) {
var result string
for len(result) < length {
num, err := rand.Int(rand.Reader, big.NewInt(int64(127)))
if err != nil {
return "", err
}
n := num.Int64()
if unicode.IsLetter(rune(n)) {
result += string(n)
}
}
return result, nil
}

193
stores/boltdb/bolt.go
View File

@ -0,0 +1,193 @@
package boltdb
import (
"bytes"
"encoding/json"
"time"
"github.com/boltdb/bolt"
"github.com/maxibanki/golang-url-shortener/stores/shared"
"github.com/pkg/errors"
)
var (
shortedURLsBucket = []byte("shorted")
shortedIDsToUserBucket = []byte("shorted2Users")
)
// BoltStore implements the stores.Storage interface
type BoltStore struct {
db *bolt.DB
}
// New returns a bolt store which implements the stores.Storage interface
func New(path string) (*BoltStore, error) {
db, err := bolt.Open(path, 0644, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, errors.Wrap(err, "could not open bolt DB database")
}
err = db.Update(func(tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists(shortedURLsBucket); err != nil {
return errors.Wrapf(err, "could not create %s bucket", shortedURLsBucket)
}
if _, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket); err != nil {
return errors.Wrapf(err, "could not create %s bucket", shortedIDsToUserBucket)
}
return err
})
if err != nil {
return nil, errors.Wrap(err, "could not create buckets")
}
return &BoltStore{
db: db,
}, nil
}
// Close closes the bolt database
func (b *BoltStore) Close() error {
return b.db.Close()
}
// GetEntryByID returns a entry and an error by the shorted ID
func (b *BoltStore) GetEntryByID(id string) (*shared.Entry, error) {
var raw []byte
err := b.db.View(func(tx *bolt.Tx) error {
raw = tx.Bucket(shortedURLsBucket).Get([]byte(id))
if raw == nil {
return shared.ErrNoEntryFound
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "could not view db")
}
var entry *shared.Entry
return entry, json.Unmarshal(raw, &entry)
}
// IncreaseVisitCounter increases the visit counter and sets the current
// time as the last visit ones
func (b *BoltStore) IncreaseVisitCounter(id string) error {
entry, err := b.GetEntryByID(id)
if err != nil {
return errors.Wrap(err, "could not get entry by ID")
}
entry.Public.VisitCount++
currentTime := time.Now()
entry.Public.LastVisit = &currentTime
raw, err := json.Marshal(entry)
if err != nil {
return errors.Wrap(err, "could not marshal json")
}
err = b.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(shortedURLsBucket).Put([]byte(id), raw); err != nil {
return errors.Wrap(err, "could not put updated visitor")
}
return nil
})
return errors.Wrap(err, "could not update entry")
}
// CreateEntry creates an entry by a given ID and returns an error
func (b *BoltStore) CreateEntry(entry shared.Entry, id, userIdentifier string) error {
entryRaw, err := json.Marshal(entry)
if err != nil {
return errors.Wrap(err, "could not marshal entry")
}
err = b.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(shortedURLsBucket)
if raw := bucket.Get([]byte(id)); raw != nil {
return errors.New("entry already exists")
}
if err := bucket.Put([]byte(id), entryRaw); err != nil {
return errors.Wrap(err, "could not put data into bucket")
}
uTsURLsBucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket)
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return uTsURLsBucket.Put([]byte(id), []byte(userIdentifier))
})
return errors.Wrap(err, "could not update db")
}
// DeleteEntry deleted an entry by a given ID and returns an error
func (b *BoltStore) DeleteEntry(id string) error {
err := b.db.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(shortedURLsBucket)
if bucket.Get([]byte(id)) == nil {
return errors.New("entry already deleted")
}
if err := bucket.Delete([]byte(id)); err != nil {
return errors.Wrap(err, "could not delete entry")
}
if err := tx.DeleteBucket([]byte(id)); err != nil && err != bolt.ErrBucketNotFound && err != bolt.ErrBucketExists {
return errors.Wrap(err, "could not delte bucket")
}
uTsIDsBucket := tx.Bucket(shortedIDsToUserBucket)
return uTsIDsBucket.ForEach(func(k, v []byte) error {
if bytes.Equal(k, []byte(id)) {
return uTsIDsBucket.Delete(k)
}
return nil
})
})
return errors.Wrap(err, "could not update db")
}
// GetVisitors returns the visitors and an error of an entry
func (b *BoltStore) GetVisitors(id string) ([]shared.Visitor, error) {
output := []shared.Visitor{}
return output, b.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(id))
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return bucket.ForEach(func(k, v []byte) error {
var value shared.Visitor
if err := json.Unmarshal(v, &value); err != nil {
return errors.Wrap(err, "could not unmarshal json")
}
output = append(output, value)
return nil
})
})
}
// GetUserEntries returns all user entries of an given user identifier
func (b *BoltStore) GetUserEntries(userIdentifier string) (map[string]shared.Entry, error) {
entries := map[string]shared.Entry{}
err := b.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket)
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
return bucket.ForEach(func(k, v []byte) error {
if bytes.Equal(v, []byte(userIdentifier)) {
entry, err := b.GetEntryByID(string(k))
if err != nil {
return errors.Wrap(err, "could not get entry")
}
entries[string(k)] = *entry
}
return nil
})
})
return entries, errors.Wrap(err, "could not update db")
}
// RegisterVisitor saves the visitor in the database
func (b *BoltStore) RegisterVisitor(id, visitID string, visitor shared.Visitor) error {
err := b.db.Update(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(id))
if err != nil {
return errors.Wrap(err, "could not create bucket")
}
data, err := json.Marshal(visitor)
if err != nil {
return errors.Wrap(err, "could not create json")
}
return bucket.Put([]byte(visitID), data)
})
return errors.Wrap(err, "could not update db")
}

46
stores/shared/shared.go
View File

@ -0,0 +1,46 @@
package shared
import (
"errors"
"time"
)
// Storage is an interface which will be implmented by each storage
// e.g. bolt, sqlite
type Storage interface {
GetEntryByID(string) (*Entry, error)
GetVisitors(string) ([]Visitor, error)
DeleteEntry(string) error
IncreaseVisitCounter(string) error
CreateEntry(Entry, string, string) error
GetUserEntries(string) (map[string]Entry, error)
RegisterVisitor(string, string, Visitor) error
Close() error
}
// Entry is the data set which is stored in the DB as JSON
type Entry struct {
OAuthProvider, OAuthID string
RemoteAddr string `json:",omitempty"`
DeletionURL string `json:",omitempty"`
Password []byte `json:",omitempty"`
Public EntryPublicData
}
// EntryPublicData is the public part of an entry
type EntryPublicData struct {
CreatedOn time.Time
LastVisit, Expiration *time.Time `json:",omitempty"`
VisitCount int
URL string
}
// Visitor is the entry which is stored in the visitors bucket
type Visitor struct {
IP, Referer, UserAgent string
Timestamp time.Time
UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm string `json:",omitempty"`
}
// ErrNoEntryFound is returned when no entry to a id is found
var ErrNoEntryFound = errors.New("no entry found with this ID")

1
stores/sqlite/sqlite.go
View File

@ -0,0 +1 @@
package sqlite

189
stores/store.go
View File

@ -0,0 +1,189 @@
// Package stores provides support to interact with the entries
package stores
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha512"
"math/big"
"path/filepath"
"strings"
"time"
"unicode"
"github.com/asaskevich/govalidator"
"github.com/maxibanki/golang-url-shortener/stores/boltdb"
"github.com/maxibanki/golang-url-shortener/stores/shared"
"github.com/maxibanki/golang-url-shortener/util"
"github.com/pborman/uuid"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
)
// Store holds internal funcs and vars about the store
type Store struct {
storage shared.Storage
idLength int
}
// ErrNoValidURL is returned when the URL is not valid
var ErrNoValidURL = errors.New("the given URL is no valid URL")
// ErrGeneratingIDFailed is returned when the 10 tries to generate an id failed
var ErrGeneratingIDFailed = errors.New("could not generate unique id, all ten tries failed")
// ErrEntryIsExpired is returned when the entry is expired
var ErrEntryIsExpired = errors.New("entry is expired")
// New initializes the store with the db
func New() (*Store, error) {
s, err := boltdb.New(filepath.Join(util.GetConfig().DataDir, "main.db"))
if err != nil {
return nil, errors.Wrap(err, "could not create bolt db store")
}
return &Store{
storage: s,
idLength: util.GetConfig().ShortedIDLength,
}, nil
}
// GetEntryByID returns a unmarshalled entry of the db by a given ID
func (s *Store) GetEntryByID(id string) (*shared.Entry, error) {
if id == "" {
return nil, shared.ErrNoEntryFound
}
return s.storage.GetEntryByID(id)
}
// GetEntryAndIncrease Increases the visitor count, checks
// if the URL is expired and returns the origin URL
func (s *Store) GetEntryAndIncrease(id string) (*shared.Entry, error) {
entry, err := s.GetEntryByID(id)
if err != nil {
return nil, err
}
if entry.Public.Expiration != nil && time.Now().After(*entry.Public.Expiration) {
return nil, ErrEntryIsExpired
}
if err := s.storage.IncreaseVisitCounter(id); err != nil {
return nil, errors.Wrap(err, "could not increase visitor counter")
}
return entry, nil
}
// CreateEntry creates a new record and returns his short id
func (s *Store) CreateEntry(entry shared.Entry, givenID, password string) (string, []byte, error) {
entry.Public.URL = strings.Replace(entry.Public.URL, " ", "%20", -1)
if !govalidator.IsURL(entry.Public.URL) {
return "", nil, ErrNoValidURL
}
if password != "" {
var err error
entry.Password, err = bcrypt.GenerateFromPassword([]byte(password), 10)
if err != nil {
return "", nil, errors.Wrap(err, "could not generate bcrypt from password")
}
}
// try it 10 times to make a short URL
for i := 1; i <= 10; i++ {
id, delID, err := s.createEntry(entry, givenID)
if err != nil && givenID != "" {
return "", nil, err
} else if err != nil {
logrus.Debugf("Could not create entry: %v", err)
continue
}
return id, delID, nil
}
return "", nil, ErrGeneratingIDFailed
}
// DeleteEntry deletes an Entry fully from the DB
func (s *Store) DeleteEntry(id string, givenHmac []byte) error {
mac := hmac.New(sha512.New, util.GetPrivateKey())
if _, err := mac.Write([]byte(id)); err != nil {
return errors.Wrap(err, "could not write hmac")
}
if !hmac.Equal(mac.Sum(nil), givenHmac) {
return errors.New("hmac verification failed")
}
return errors.Wrap(s.storage.DeleteEntry(id), "could not delete entry")
}
// RegisterVisit registers an new incoming request in the store
func (s *Store) RegisterVisit(id string, visitor shared.Visitor) {
requestID := uuid.New()
logrus.WithFields(logrus.Fields{
"ClientIP": visitor.IP,
"ID": id,
"RequestID": requestID,
}).Info("New redirect was registered...")
if err := s.storage.RegisterVisitor(id, requestID, visitor); err != nil {
logrus.Warningf("could not register visit: %v", err)
}
}
// GetVisitors returns all the visits of a shorted URL
func (s *Store) GetVisitors(id string) ([]shared.Visitor, error) {
visitors, err := s.storage.GetVisitors(id)
if err != nil {
return nil, errors.Wrap(err, "could not get visitors")
}
return visitors, nil
}
// GetUserEntries returns all the shorted URL entries of an user
func (s *Store) GetUserEntries(oAuthProvider, oAuthID string) (map[string]shared.Entry, error) {
userIdentifier := getUserIdentifier(oAuthProvider, oAuthID)
entries, err := s.storage.GetUserEntries(userIdentifier)
if err != nil {
return nil, errors.Wrap(err, "could not get user entries")
}
return entries, nil
}
func getUserIdentifier(oAuthProvider, oAuthID string) string {
return oAuthProvider + oAuthID
}
// Close closes the bolt db database
func (s *Store) Close() error {
return s.storage.Close()
}
// createEntry creates a new entry with a randomly generated id. If on is present
// then the given ID is used
func (s *Store) createEntry(entry shared.Entry, entryID string) (string, []byte, error) {
var err error
if entryID == "" {
if entryID, err = generateRandomString(s.idLength); err != nil {
return "", nil, errors.Wrap(err, "could not generate random string")
}
}
entry.Public.CreatedOn = time.Now()
mac := hmac.New(sha512.New, util.GetPrivateKey())
if _, err := mac.Write([]byte(entryID)); err != nil {
return "", nil, errors.Wrap(err, "could not write hmac")
}
if err := s.storage.CreateEntry(entry, entryID, getUserIdentifier(entry.OAuthProvider, entry.OAuthID)); err != nil {
return "", nil, errors.Wrap(err, "could not create entry")
}
return entryID, mac.Sum(nil), nil
}
// generateRandomString generates a random string with an predefined length
func generateRandomString(length int) (string, error) {
var result string
for len(result) < length {
num, err := rand.Int(rand.Reader, big.NewInt(int64(127)))
if err != nil {
return "", err
}
n := num.Int64()
if unicode.IsLetter(rune(n)) {
result += string(n)
}
}
return result, nil
}

60
store/store_test.go → stores/store_test.go
View File

@ -1,9 +1,10 @@
package store package stores
import ( import (
"strings" "strings"
"testing" "testing"
"github.com/maxibanki/golang-url-shortener/stores/shared"
"github.com/maxibanki/golang-url-shortener/util" "github.com/maxibanki/golang-url-shortener/util"
) )
@ -55,13 +56,13 @@ func TestCreateEntry(t *testing.T) {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
defer cleanup(store) defer cleanup(store)
_, _, err = store.CreateEntry(Entry{}, "", "") _, _, err = store.CreateEntry(shared.Entry{}, "", "")
if err != ErrNoValidURL { if err != ErrNoValidURL {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
for i := 1; i <= 100; i++ { for i := 1; i <= 100; i++ {
_, _, err := store.CreateEntry(Entry{ _, _, err := store.CreateEntry(shared.Entry{
Public: EntryPublicData{ Public: shared.EntryPublicData{
URL: "https://golang.org/", URL: "https://golang.org/",
}, },
}, "", "") }, "", "")
@ -78,45 +79,12 @@ func TestGetEntryByID(t *testing.T) {
} }
defer cleanup(store) defer cleanup(store)
_, err = store.GetEntryByID("something that not exists") _, err = store.GetEntryByID("something that not exists")
if err != ErrNoEntryFound { if !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) {
t.Fatalf("could not get expected '%v' error: %v", ErrNoEntryFound, err) t.Fatalf("could not get expected '%v' error: %v", shared.ErrNoEntryFound, err)
} }
_, err = store.GetEntryByID("") _, err = store.GetEntryByID("")
if err != ErrNoEntryFound { if !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) {
t.Fatalf("could not get expected '%v' error: %v", ErrNoEntryFound, err) t.Fatalf("could not get expected '%v' error: %v", shared.ErrNoEntryFound, err)
}
}
func TestIncreaseVisitCounter(t *testing.T) {
store, err := New()
if err != nil {
t.Fatalf("could not create store: %v", err)
}
defer cleanup(store)
id, _, err := store.CreateEntry(Entry{
Public: EntryPublicData{
URL: "https://golang.org/",
},
}, "", "")
if err != nil {
t.Fatalf("could not create entry: %v", err)
}
entryBeforeInc, err := store.GetEntryByID(id)
if err != nil {
t.Fatalf("could not get entry by id: %v", err)
}
if err = store.IncreaseVisitCounter(id); err != nil {
t.Fatalf("could not increase visit counter %v", err)
}
entryAfterInc, err := store.GetEntryByID(id)
if err != nil {
t.Fatalf("could not get entry by id: %v", err)
}
if entryBeforeInc.Public.VisitCount+1 != entryAfterInc.Public.VisitCount {
t.Fatalf("the increasement was not successful, the visit count is not correct")
}
if err = store.IncreaseVisitCounter(""); !strings.Contains(err.Error(), ErrNoEntryFound.Error()) {
t.Fatalf("could not get expected '%v'; got: %v", ErrNoEntryFound, err)
} }
} }
@ -126,8 +94,8 @@ func TestDelete(t *testing.T) {
t.Fatalf("could not create store: %v", err) t.Fatalf("could not create store: %v", err)
} }
defer cleanup(store) defer cleanup(store)
entryID, delHMac, err := store.CreateEntry(Entry{ entryID, delHMac, err := store.CreateEntry(shared.Entry{
Public: EntryPublicData{ Public: shared.EntryPublicData{
URL: "https://golang.org/", URL: "https://golang.org/",
}, },
}, "", "") }, "", "")
@ -137,7 +105,7 @@ func TestDelete(t *testing.T) {
if err := store.DeleteEntry(entryID, delHMac); err != nil { if err := store.DeleteEntry(entryID, delHMac); err != nil {
t.Fatalf("could not delete entry: %v", err) t.Fatalf("could not delete entry: %v", err)
} }
if _, err := store.GetEntryByID(entryID); err != ErrNoEntryFound { if _, err := store.GetEntryByID(entryID); !strings.Contains(err.Error(), shared.ErrNoEntryFound.Error()) {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
} }
@ -149,8 +117,8 @@ func TestGetURLAndIncrease(t *testing.T) {
} }
defer cleanup(store) defer cleanup(store)
const url = "https://golang.org/" const url = "https://golang.org/"
entryID, _, err := store.CreateEntry(Entry{ entryID, _, err := store.CreateEntry(shared.Entry{
Public: EntryPublicData{ Public: shared.EntryPublicData{
URL: url, URL: url,
}, },
}, "", "") }, "", "")

2
util/config.go
View File

@ -51,7 +51,7 @@ func ReadInConfig() error {
} else if !os.IsNotExist(err) { } else if !os.IsNotExist(err) {
return errors.Wrap(err, "could not read config file") return errors.Wrap(err, "could not read config file")
} else { } else {
logrus.Info("No configuration file found, using defaults and environment variable overrides.") logrus.Info("No configuration file found, using defaults with environment variable overrides.")
} }
if err := config.ApplyEnvironmentConfig(); err != nil { if err := config.ApplyEnvironmentConfig(); err != nil {
return errors.Wrap(err, "could not apply environment configuration") return errors.Wrap(err, "could not apply environment configuration")

Write Preview
Loading…
Cancel
Save