Added redis support (#100)

- add a redis store implementing stores.Storage
- add config file support to pick a storage backend
- add config file support to set redis host:port and password
- add docker_releases to .gitignore
- update README to mention redis support
- update example config.yaml

.gitignore

@@ -24,3 +24,4 @@ debug.test
 /store/main.db
 /releases
 /data
+docker_releases/

README.md

@@ -15,9 +15,11 @@
 - Expirable Links
 - URL deletion
 - Authorization System via OAuth 2.0 (Google, GitHub and Microsoft)
-- High performance database with [bolt](https://github.com/boltdb/bolt)
 - Easy [ShareX](https://github.com/ShareX/ShareX) integration
 - Dockerizable
+- Multiple supported storage backends
+    - High performance local database with [bolt](https://github.com/boltdb/bolt)
+    - Persistent non-local storage with [redis](https://redis.io/)
 
 ## [Webinterface](https://so.sh0rt.cat)
 

build/config.yaml

@@ -1,5 +1,8 @@
 ListenAddr: ':8080' # Consists of 'IP:Port', e.g. ':8080' listens on any IP and on Port 8080
 BaseURL: 'http://localhost:3000' # Origin URL, required for the authentication via OAuth
+Backend: boltdb # Can be 'boltdb' or 'redis'
+RedisHost: localhost:6379 # If using the redis backend, a host:port combination
+RedisPassword: replace me # if using the redis backend, a conneciton password.
 DataDir: ./data # Contains: the database and the private key
 EnableDebugMode: true # Activates more detailed logging
 ShortedIDLength: 10 # Length of the random generated ID which is used for new shortened URLs

stores/redis/redis.go

@@ -0,0 +1,351 @@
+package redis
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/go-redis/redis"
+	"github.com/mxschmitt/golang-url-shortener/stores/shared"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	entryPathPrefix     = "entry:"       // prefix for path-to-url mappings
+	entryUserPrefix     = "user:"        // prefix for path-to-user mappings
+	userToEntriesPrefix = "userEntries:" // prefix for user-to-[]entries mappings (redis SET)
+	entryVisitsPrefix   = "entryVisits:" // prefix for entry-to-[]visit mappings (redis LIST)
+)
+
+// Store implements the stores.Storage interface
+type Store struct {
+	c *redis.Client
+}
+
+// New initializes connection to the redis instance.
+func New(hostaddr, password string) (*Store, error) {
+	c := redis.NewClient(&redis.Options{
+		Addr:     hostaddr,
+		Password: password,
+		DB:       0,
+	})
+	// if we can't talk to redis, fail fast
+	_, err := c.Ping().Result()
+	if err != nil {
+		return nil, errors.Wrap(err, "Could not connect to redis db0")
+	}
+	ret := &Store{c: c}
+	return ret, nil
+}
+
+// keyExists checks for the existence of a key in redis.
+func (r *Store) keyExists(key string) (exists bool, err error) {
+	logrus.Debugf("Checking for existence of key: %s", key)
+	result := r.c.Exists(key)
+	if result.Err() != nil {
+		msg := fmt.Sprintf("Error looking up key '%s': '%v', got val: '%d'", key, result.Err(), result.Val())
+		logrus.Error(msg)
+		return false, errors.Wrap(result.Err(), msg)
+	}
+	if result.Val() == 1 {
+		logrus.Debugf("Key '%s' exists!", key)
+		return true, nil
+	}
+	logrus.Debugf("Key '%s' does not exist!", key)
+	return false, nil
+}
+
+// setValue sets the value of a key in redis.
+func (r *Store) setValue(key string, raw []byte) error {
+	logrus.Debugf("Setting value for key '%s: '%s''", key, raw)
+	status := r.c.Set(key, raw, 0) // n.b. expiration 0 means never expire
+	if status.Err() != nil {
+		msg := fmt.Sprintf("Got an unexpected error adding key '%s': %s", key, status.Err())
+		logrus.Error(msg)
+		return errors.Wrap(status.Err(), msg)
+	}
+	return nil
+}
+
+// createValue is a wrapper around setValue that returns an error if the key already exists.
+func (r *Store) createValue(key string, raw []byte) error {
+	logrus.Debugf("Creating key '%s'", key)
+	exists, err := r.keyExists(key)
+	if err != nil {
+		msg := fmt.Sprintf("Could not check existence of key '%s': %s", key, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	if exists == true {
+		msg := fmt.Sprintf("Could not create key '%s':  already exists", key)
+		logrus.Error(msg)
+		return errors.New(msg)
+	}
+	return r.setValue(key, raw)
+}
+
+// delValue deletes a key in redis.
+func (r *Store) delValue(key string) error {
+	logrus.Debugf("Deleting key '%s'", key)
+
+	exists, err := r.keyExists(key)
+	if err != nil {
+		msg := fmt.Sprintf("Could not check existence of key '%s': %s", key, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	if exists == false {
+		logrus.Warnf("Tried to delete key '%s' but it's already gone", key)
+		return err
+	}
+
+	status := r.c.Del(key)
+	if status.Err() != nil {
+		msg := fmt.Sprintf("Got an unexpected error deleting key '%s': %s", key, status.Err())
+		logrus.Error(msg)
+		return errors.Wrap(status.Err(), msg)
+	}
+	return err
+}
+
+// CreateEntry creates an entry (path->url mapping) and all associated stored data.
+func (r *Store) CreateEntry(entry shared.Entry, id, userIdentifier string) error {
+	// add the entry (path->url mapping)
+	logrus.Debugf("Creating entry '%s' for user '%s'", id, userIdentifier)
+	raw, err := json.Marshal(entry)
+	if err != nil {
+		msg := fmt.Sprintf("Could not marshal JSON for entry %s: %v", id, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	entryKey := entryPathPrefix + id
+	logrus.Debugf("Adding key '%s': %s", entryKey, raw)
+	err = r.createValue(entryKey, raw)
+	if err != nil {
+		msg := fmt.Sprintf("Failed to set key '%s' for user '%s': %v", entryKey, userIdentifier, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	// add the path->user mapping
+	userKey := entryUserPrefix + id
+	logrus.Debugf("Adding key '%s': %s", userKey, raw)
+	err = r.createValue(userKey, []byte(userIdentifier))
+	if err != nil {
+		msg := fmt.Sprintf("Failed to set key '%s' for user '%s': %v", userKey, userIdentifier, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	// add the entry to the SET of entries for the useridentifier
+	userEntriesKey := userToEntriesPrefix + userIdentifier
+	logrus.Debugf("Adding entry '%s' to set of entries for user '%s'", id, userIdentifier)
+	result := r.c.SAdd(userEntriesKey, id)
+	if result.Err() != nil {
+		msg := fmt.Sprintf("Failed to add entry '%s' for user '%s': %v", id, userIdentifier, result.Err())
+		logrus.Error(msg)
+		return errors.Wrap(result.Err(), msg)
+	}
+	logrus.Debugf("Successfully added entry '%s' to set '%s'", id, userEntriesKey)
+	return nil
+}
+
+// DeleteEntry deletes an entry and all associated stored data.
+func (r *Store) DeleteEntry(id string) error {
+	// delete the id-to-url mapping
+	entryKey := entryPathPrefix + id
+	err := r.delValue(entryKey)
+	if err != nil {
+		msg := fmt.Sprintf("Could not delete entry id %s: %v", id, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	// delete the visitors list for the id
+	entryVisitsKey := entryVisitsPrefix + id
+	err = r.delValue(entryVisitsKey)
+	if err != nil {
+		msg := fmt.Sprintf("Could not delete visitors list for id %s: %v", id, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	// get the user for the id
+	userKey := entryUserPrefix + id
+	var userIdentifier string
+	userIdentifier, err = r.c.Get(userKey).Result()
+	if err != nil {
+		msg := fmt.Sprintf("Could not fetch id to user mapping for id '%s': %v", id, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	// delete the entry from set of entries for the user
+	userEntriesKey := userToEntriesPrefix + userIdentifier
+	err = r.c.SRem(userEntriesKey, id).Err()
+	if err != nil {
+		msg := fmt.Sprintf("Could not remove entry '%s' from list of entries for user '%s': %v", id, userIdentifier, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	// delete the id-to-user mapping
+	err = r.delValue(userKey)
+	if err != nil {
+		msg := fmt.Sprintf("Could not delete the path-to-user mapping for entry '%s': %v", id, err)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+
+	return err
+}
+
+// GetEntryByID looks up an entry by its path and returns a pointer to a
+// shared.Entry instance, with the visit count and last visit time set
+// properly.
+func (r *Store) GetEntryByID(id string) (*shared.Entry, error) {
+	entryKey := entryPathPrefix + id
+	logrus.Debugf("Fetching key: '%s'", entryKey)
+	result := r.c.Get(entryKey)
+	raw, err := result.Bytes()
+	if err != nil {
+		msg := fmt.Sprintf("Error looking up key '%s': %s'", entryKey, err)
+		logrus.Warn(msg)
+		err = shared.ErrNoEntryFound
+		return nil, err
+	}
+	logrus.Debugf("Got entry for key '%s': '%s'", entryKey, raw)
+
+	var entry *shared.Entry
+	err = json.Unmarshal(raw, &entry)
+	if err != nil {
+		msg := fmt.Sprintf("Error unmarshalling JSON for entry '%s': %v  (json str: '%s')", id, err, raw)
+		logrus.Error(msg)
+		return nil, errors.Wrap(err, msg)
+	}
+
+	// now we interleave the visit count and the last visit time
+	// from the redis sources (we do this so we don't have to rewrite
+	// the entry every time someone visits which is madness)
+	//
+	// first, the visit count is just the length of the visitors list
+	entryVisitsKey := entryVisitsPrefix + id
+	visitCount, err := r.c.LLen(entryVisitsKey).Result()
+	if err != nil {
+		logrus.Warnf("Could not get length of visitor list for id '%s': '%v'", id, err)
+		entry.Public.VisitCount = int(0) // or zero if nobody's visited, that's fine.
+	} else {
+		entry.Public.VisitCount = int(visitCount)
+	}
+
+	// grab the timestamp out of the last visitor on the list
+	var visitor *shared.Visitor
+	lastVisit := time.Time(time.Unix(0, 0)) // default to start-of-epoch if we can't figure it out
+	raw, err = r.c.LIndex(entryVisitsKey, 0).Bytes()
+	if err != nil {
+		logrus.Warnf("Could not fetch visitor list for entry '%s': %v", id, err)
+	} else {
+		err = json.Unmarshal(raw, &visitor)
+		if err != nil {
+			logrus.Warnf("Could not unmarshal JSON for last visitor to entry '%s': %v  (got string: '%s')", id, err, raw)
+		} else {
+			lastVisit = visitor.Timestamp
+		}
+	}
+	logrus.Debugf("Setting last visit time for entry '%s' to '%v'", id, lastVisit)
+	entry.Public.LastVisit = &lastVisit
+
+	return entry, nil
+}
+
+// GetUserEntries returns all entries that are owned by a given user, in the
+// form of a map of path->shared.Entry
+func (r *Store) GetUserEntries(userIdentifier string) (map[string]shared.Entry, error) {
+	logrus.Debugf("Getting all entries for user %s", userIdentifier)
+	entries := map[string]shared.Entry{}
+	key := userToEntriesPrefix + userIdentifier
+	result := r.c.SMembers(key)
+	if result.Err() != nil {
+		msg := fmt.Sprintf("Could not fetch set of entries for user '%s': %v", userIdentifier, result.Err())
+		logrus.Errorf(msg)
+		return nil, errors.Wrap(result.Err(), msg)
+	}
+	for _, v := range result.Val() {
+		logrus.Debugf("got entry: %s", v)
+		entry, err := r.GetEntryByID(string(v))
+		if err != nil {
+			msg := fmt.Sprintf("Could not get entry '%s': %s", v, err)
+			logrus.Warn(msg)
+		} else {
+			entries[string(v)] = *entry
+		}
+	}
+	logrus.Debugf("all out of entries")
+	return entries, nil
+}
+
+// RegisterVisitor adds a shared.Visitor to the list of visits for a path.
+func (r *Store) RegisterVisitor(id, visitId string, visitor shared.Visitor) error {
+	data, err := json.Marshal(visitor)
+	if err != nil {
+		msg := fmt.Sprintf("Could not marshal JSON for entry %s, visitId %s", id, visitId)
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	// push the visit data onto a redis list who's key is the url id
+	key := entryVisitsPrefix + id
+	result := r.c.LPush(key, data)
+	if result.Err() != nil {
+		msg := fmt.Sprintf("Could not register visitor for ID %s", id)
+		logrus.Error(msg)
+		return errors.Wrap(result.Err(), msg)
+	}
+	return err
+}
+
+// Return the full list of visitors for a path.
+func (r *Store) GetVisitors(id string) ([]shared.Visitor, error) {
+	var visitors []shared.Visitor
+	key := entryVisitsPrefix + id
+	// TODO: for non-trivial numbers of keys, this could start
+	// to get hairy; should convert to a paginated Scan operation.
+	result := r.c.LRange(key, 0, -1)
+	if result.Err() != nil {
+		msg := fmt.Sprintf("Could not get visitors for id '%s'", id)
+		logrus.Error(msg)
+		return nil, errors.Wrap(result.Err(), msg)
+	}
+	for _, v := range result.Val() {
+		var value shared.Visitor
+		if err := json.Unmarshal([]byte(v), &value); err != nil {
+			msg := fmt.Sprintf("Could not unmarshal json for visit '%s': %v", id, err)
+			logrus.Error(msg)
+			return nil, errors.Wrap(err, msg)
+		}
+		visitors = append(visitors, value)
+	}
+	return visitors, nil
+}
+
+// IncreaseVisitCounter is a no-op and returns nil for all values.
+//
+// This function is unnecessary for the redis backend: we already
+// have a redis LIST of visitors, and we can derive the visit count
+// by calling redis.client.LLen(list) (which is a constant-time op)
+// during GetEntryByID().  If we want the timestamp of the most recent
+// visit we can pull the most recent visit off with redis.client.LIndex(0)
+// (also constant-time) and reading the timetamp field.
+func (r *Store) IncreaseVisitCounter(id string) error {
+	return nil
+}
+
+// Close closes the connection to redis.
+func (r *Store) Close() error {
+	err := r.c.Close()
+	if err != nil {
+		msg := "Cloud not close the redis connection"
+		logrus.Error(msg)
+		return errors.Wrap(err, msg)
+	}
+	return err
+}

stores/store.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/asaskevich/govalidator"
 	"github.com/mxschmitt/golang-url-shortener/stores/boltdb"
+	"github.com/mxschmitt/golang-url-shortener/stores/redis"
 	"github.com/mxschmitt/golang-url-shortener/stores/shared"
 	"github.com/mxschmitt/golang-url-shortener/util"
 	"github.com/pborman/uuid"
@@ -38,9 +39,18 @@ var ErrEntryIsExpired = errors.New("entry is expired")
 
 // New initializes the store with the db
 func New() (*Store, error) {
-	s, err := boltdb.New(filepath.Join(util.GetConfig().DataDir, "main.db"))
+	var err error
+	var s shared.Storage
+	switch backend := util.GetConfig().Backend; backend {
+	case "redis":
+		s, err = redis.New(util.GetConfig().RedisHost, util.GetConfig().RedisPassword)
+	case "boltdb":
+		s, err = boltdb.New(filepath.Join(util.GetConfig().DataDir, "main.db"))
+	default:
+		return nil, errors.New(backend + " is not a recognized backend")
+	}
 	if err != nil {
-		return nil, errors.Wrap(err, "could not create bolt db store")
+		return nil, errors.Wrap(err, "could not initialize the data backend")
 	}
 	return &Store{
 		storage:  s,
@@ -61,6 +71,7 @@ func (s *Store) GetEntryByID(id string) (*shared.Entry, error) {
 func (s *Store) GetEntryAndIncrease(id string) (*shared.Entry, error) {
 	entry, err := s.GetEntryByID(id)
 	if err != nil {
+		errors.Wrap(err, "could not fetch entry "+id)
 		return nil, err
 	}
 	if entry.Public.Expiration != nil && time.Now().After(*entry.Public.Expiration) {

stores/store_test.go

@@ -65,6 +65,7 @@ func TestGenerateRandomString(t *testing.T) {
 func TestStore(t *testing.T) {
 	util.SetConfig(util.Configuration{
 		DataDir:         testData.DataDir,
+		Backend:         "boltdb",
 		ShortedIDLength: 4,
 	})
 	if err := os.MkdirAll(testData.DataDir, 0755); err != nil {

util/config.go

@@ -17,6 +17,9 @@ type Configuration struct {
 	ListenAddr      string    `yaml:"ListenAddr" env:"LISTEN_ADDR"`
 	BaseURL         string    `yaml:"BaseURL" env:"BASE_URL"`
 	DataDir         string    `yaml:"DataDir" env:"DATA_DIR"`
+	Backend         string    `yaml:"Backend" env:"BACKEND"`
+	RedisHost       string    `yaml:"RedisHost" env:"REDIS_HOST"`
+	RedisPassword   string    `yaml:"RedisPassword" env:"REDIS_PASSWORD"`
 	UseSSL          bool      `yaml:"EnableSSL" env:"USE_SSL"`
 	EnableDebugMode bool      `yaml:"EnableDebugMode" env:"ENABLE_DEBUG_MODE"`
 	ShortedIDLength int       `yaml:"ShortedIDLength" env:"SHORTED_ID_LENGTH"`
@@ -35,6 +38,7 @@ var config = Configuration{
 	ListenAddr:      ":8080",
 	BaseURL:         "http://localhost:3000",
 	DataDir:         "data",
+	Backend:         "boltdb",
 	EnableDebugMode: false,
 	UseSSL:          false,
 	ShortedIDLength: 4,

util/test.yaml

@@ -1,4 +1,5 @@
 listen_addr: ':8080'
+backend: 'boltdb'
 base_url: 'http://localhost:3000'
 data_dir: ./data
 enable_debug_mode: true