From ac05d6f03647ed1f49cea9081ac867c4a43861a0 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Sat, 25 Nov 2017 00:06:41 +0100 Subject: [PATCH] Minor changes: - Added recent shorted URLs listing - Added visitor page of a shorted URL #43 --- handlers/handlers.go | 2 + handlers/public.go | 46 +++++++++++- main_test.go | 4 +- static/public/index.html | 4 +- static/src/Recent/Recent.js | 52 ++++++++++++++ static/src/Visitors/Visitors.js | 63 +++++++++++++++++ static/src/index.js | 6 +- store/store.go | 122 +++++++++++++++++++++++++++----- store/util.go | 12 ++-- 9 files changed, 283 insertions(+), 28 deletions(-) create mode 100644 static/src/Recent/Recent.js create mode 100644 static/src/Visitors/Visitors.js diff --git a/handlers/handlers.go b/handlers/handlers.go index 6e2027f..8c6cb22 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -71,6 +71,8 @@ func (h *Handler) setHandlers() error { protected.Use(h.authMiddleware) protected.POST("/create", h.handleCreate) protected.POST("/lookup", h.handleLookup) + protected.POST("/recent", h.handleRecent) + protected.POST("/visitors", h.handleGetVisitors) h.engine.GET("/api/v1/info", h.handleInfo) h.engine.GET("/d/:id/:hash", h.handleDelete) diff --git a/handlers/public.go b/handlers/public.go index 41e8ee9..a18cf08 100644 --- a/handlers/public.go +++ b/handlers/public.go @@ -19,7 +19,7 @@ import ( type urlUtil struct { URL string `binding:"required"` ID, DeletionURL string - Expiration *time.Time `json:",omitempty"` + Expiration *time.Time } // handleLookup is the http handler for getting the infos @@ -49,14 +49,28 @@ func (h *Handler) handleLookup(c *gin.Context) { // handleAccess handles the access for incoming requests func (h *Handler) handleAccess(c *gin.Context) { - url, err := h.store.GetURLAndIncrease(c.Request.URL.Path[1:]) + id := c.Request.URL.Path[1:] + url, err := h.store.GetURLAndIncrease(id) if err == store.ErrNoEntryFound { return } else if err != nil { http.Error(c.Writer, fmt.Sprintf("could not get and crease visitor counter: %v, ", err), http.StatusInternalServerError) return } + go h.store.RegisterVisit(id, store.Visitor{ + IP: c.ClientIP(), + Timestamp: time.Now(), + Referer: c.GetHeader("Referer"), + UTMSource: c.Query("utm_source"), + UTMMedium: c.Query("utm_medium"), + UTMCampaign: c.Query("utm_campaign"), + UTMContent: c.Query("utm_content"), + UTMTerm: c.Query("utm_term"), + }) c.Redirect(http.StatusTemporaryRedirect, url) + // There is a need to Abort in the current middleware to prevent + // that the status code will be overridden by the default NoRoute handler + c.Abort() } // handleCreate handles requests to create an entry @@ -87,6 +101,23 @@ func (h *Handler) handleCreate(c *gin.Context) { }) } +// handleGetVisitors handles requests to create an entry +func (h *Handler) handleGetVisitors(c *gin.Context) { + var data struct { + ID string `binding:"required"` + } + if err := c.ShouldBind(&data); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + dataSets, err := h.store.GetVisitors(data.ID) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, dataSets) +} + func (h *Handler) handleInfo(c *gin.Context) { info := gin.H{ "providers": h.providers, @@ -97,6 +128,17 @@ func (h *Handler) handleInfo(c *gin.Context) { } c.JSON(http.StatusOK, info) } + +func (h *Handler) handleRecent(c *gin.Context) { + user := c.MustGet("user").(*auth.JWTClaims) + entries, err := h.store.GetUserEntries(user.OAuthProvider, user.OAuthID) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, entries) +} + func (h *Handler) handleDelete(c *gin.Context) { givenHmac, err := base64.RawURLEncoding.DecodeString(c.Param("hash")) if err != nil { diff --git a/main_test.go b/main_test.go index 18b5488..92aa338 100644 --- a/main_test.go +++ b/main_test.go @@ -2,7 +2,6 @@ package main import ( "net" - "os" "testing" "time" @@ -14,11 +13,10 @@ func TestInitShortener(t *testing.T) { if err != nil { t.Fatalf("could not init shortener: %v", err) } - time.Sleep(time.Second) // Give the http server a second to boot up + time.Sleep(time.Millisecond * 200) // Give the http server a second to boot up // We expect there a port is in use error if _, err := net.Listen("tcp", viper.GetString("listen_addr")); err == nil { t.Fatalf("port is not in use: %v", err) } close() - os.Exit(0) } diff --git a/static/public/index.html b/static/public/index.html index 8ebdd5d..ba432e4 100644 --- a/static/public/index.html +++ b/static/public/index.html @@ -19,7 +19,9 @@ work correctly both with client-side routing and a non-root public URL. Learn how to configure a non-root public URL by running `npm run build`. --> - URL Shortener + Golang URL Shortener + + diff --git a/static/src/Recent/Recent.js b/static/src/Recent/Recent.js new file mode 100644 index 0000000..77f4d34 --- /dev/null +++ b/static/src/Recent/Recent.js @@ -0,0 +1,52 @@ +import React, { Component } from 'react' +import { Container, Table } from 'semantic-ui-react' +import toastr from 'toastr' +import moment from 'moment' + +export default class RecentComponent extends Component { + state = { + recent: null + } + + componentWillMount() { + fetch('/api/v1/protected/recent', { + method: 'POST', + headers: { + 'Authorization': window.localStorage.getItem('token'), + } + }) + .then(res => res.ok ? res.json() : Promise.reject(res.json())) + .then(recent => this.setState({ recent: recent })) + .catch(e => e.done(res => toastr.error(`Could not fetch recent shortened URLs: ${res}`))) + } + + onRowClick(id) { + this.props.history.push(`/visitors/${id}`) + } + + render() { + const { recent } = this.state + return ( + + + + + Original URL + Created + Short URL + All Clicks + + + + {recent && Object.keys(recent).map(key => + {recent[key].Public.URL} + {moment(recent[key].Public.CreatedOn).format('LLL')} + {`${window.location.origin}/${key}`} + {recent[key].Public.VisitCount} + )} + +
+ + ) + } +}; diff --git a/static/src/Visitors/Visitors.js b/static/src/Visitors/Visitors.js new file mode 100644 index 0000000..8d9f41a --- /dev/null +++ b/static/src/Visitors/Visitors.js @@ -0,0 +1,63 @@ +import React, { Component } from 'react' +import { Container, Table } from 'semantic-ui-react' +import moment from 'moment' +import toastr from 'toastr' + +export default class VisitorComponent extends Component { + state = { + id: "", + visitors: null + } + + componentDidMount() { + this.setState({ id: this.props.match.params.id }) + fetch('/api/v1/protected/visitors', { + method: 'POST', + body: JSON.stringify({ + ID: this.props.match.params.id + }), + headers: { + 'Authorization': window.localStorage.getItem('token'), + 'Content-Type': 'application/json' + } + }) + .then(res => res.ok ? res.json() : Promise.reject(res.json())) + .then(visitors => this.setState({ visitors })) + .catch(e => e.done(res => toastr.error(`Could not fetch visitors: ${res}`))) + } + + // getUTMSource is a function which generates the output for the utm[...] table column + getUTMSource(visit) { + return [visit.UTMSource, visit.UTMMedium, visit.UTMCampaign, visit.UTMContent, visit.UTMTerm] + .map(value => value ? value : null) + .filter(v => v) + .map((value, idx, data) => value + (idx !== data.length - 1 ? ", " : "")) + .join("") + } + + render() { + const { visitors } = this.state + return ( + + + + + Timestamp + IP + Referer + UTM (source, medium, campaign, content, term) + + + + {visitors && visitors.map((visit, idx) => + {moment(visit.Timestamp).format('LLL')} + {visit.IP} + {visit.Referer} + {this.getUTMSource(visit)} + )} + +
+ + ) + } +}; diff --git a/static/src/index.js b/static/src/index.js index 788d2e0..db81965 100644 --- a/static/src/index.js +++ b/static/src/index.js @@ -10,6 +10,8 @@ import About from './About/About' import Home from './Home/Home' import ShareX from './ShareX/ShareX' import Lookup from './Lookup/Lookup' +import Recent from './Recent/Recent' +import Visitors from './Visitors/Visitors' export default class BaseComponent extends Component { state = { @@ -22,7 +24,7 @@ export default class BaseComponent extends Component { handleItemClick = (e, { name }) => this.setState({ activeItem: name }) - onOAuthClose() { + onOAuthClose = () => { this.setState({ oAuthOpen: true }) } @@ -159,6 +161,8 @@ export default class BaseComponent extends Component { + + ) diff --git a/store/store.go b/store/store.go index c9a2af6..6ad1a0a 100644 --- a/store/store.go +++ b/store/store.go @@ -2,6 +2,7 @@ package store import ( + "bytes" "crypto/hmac" "crypto/sha512" "encoding/json" @@ -9,6 +10,7 @@ import ( "time" "github.com/maxibanki/golang-url-shortener/util" + "github.com/pborman/uuid" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -19,9 +21,8 @@ import ( // Store holds internal funcs and vars about the store type Store struct { - db *bolt.DB - bucketName []byte - idLength int + db *bolt.DB + idLength int } // Entry is the data set which is stored in the DB as JSON @@ -31,12 +32,19 @@ type Entry struct { Public EntryPublicData } +// Visitor is the entry which is stored in the visitors bucket +type Visitor struct { + IP, Referer string + Timestamp time.Time + UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm string `json:",omitempty"` +} + // EntryPublicData is the public part of an entry type EntryPublicData struct { - CreatedOn, LastVisit time.Time - Expiration *time.Time `json:",omitempty"` - VisitCount int - URL string + CreatedOn time.Time + LastVisit, Expiration *time.Time `json:",omitempty"` + VisitCount int + URL string } // ErrNoEntryFound is returned when no entry to a id is found @@ -51,24 +59,27 @@ var ErrGeneratingIDFailed = errors.New("could not generate unique id, all ten tr // ErrEntryIsExpired is returned when the entry is expired var ErrEntryIsExpired = errors.New("entry is expired") +var ( + shortedURLsBucket = []byte("shorted") + shortedIDsToUserBucket = []byte("shorted2IDs") +) + // New initializes the store with the db func New() (*Store, error) { db, err := bolt.Open(filepath.Join(util.GetDataDir(), "main.db"), 0644, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { return nil, errors.Wrap(err, "could not open bolt DB database") } - bucketName := []byte("shorted") err = db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(bucketName) + _, err := tx.CreateBucketIfNotExists(shortedURLsBucket) return err }) if err != nil { return nil, err } return &Store{ - db: db, - idLength: viper.GetInt("shorted_id_length"), - bucketName: bucketName, + db: db, + idLength: viper.GetInt("shorted_id_length"), }, nil } @@ -92,13 +103,14 @@ func (s *Store) IncreaseVisitCounter(id string) error { return errors.Wrap(err, "could not get entry by ID") } entry.Public.VisitCount++ - entry.Public.LastVisit = time.Now() + currentTime := time.Now() + entry.Public.LastVisit = ¤tTime raw, err := json.Marshal(entry) if err != nil { return err } return s.db.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket(s.bucketName).Put([]byte(id), raw); err != nil { + if err := tx.Bucket(shortedURLsBucket).Put([]byte(id), raw); err != nil { return errors.Wrap(err, "could not put updated visitor count JSON into the bucket") } return nil @@ -125,7 +137,7 @@ func (s *Store) GetURLAndIncrease(id string) (string, error) { func (s *Store) GetEntryByIDRaw(id string) ([]byte, error) { var raw []byte return raw, s.db.View(func(tx *bolt.Tx) error { - raw = tx.Bucket(s.bucketName).Get([]byte(id)) + raw = tx.Bucket(shortedURLsBucket).Get([]byte(id)) if raw == nil { return ErrNoEntryFound } @@ -162,11 +174,87 @@ func (s *Store) DeleteEntry(id string, givenHmac []byte) error { return errors.New("hmac verification failed") } return s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(s.bucketName) + bucket := tx.Bucket(shortedURLsBucket) if bucket.Get([]byte(id)) == nil { return errors.New("entry already deleted") } - return bucket.Delete([]byte(id)) + if err := bucket.Delete([]byte(id)); err != nil { + return errors.Wrap(err, "could not delete entry") + } + if err := tx.DeleteBucket([]byte(id)); err != nil && err != bolt.ErrBucketNotFound && err != bolt.ErrBucketExists { + return errors.Wrap(err, "could not delte bucket") + } + uTsIDsBucket := tx.Bucket(shortedIDsToUserBucket) + return uTsIDsBucket.ForEach(func(k, v []byte) error { + if bytes.Equal(k, []byte(id)) { + return uTsIDsBucket.Delete(k) + } + return nil + }) + }) +} + +// RegisterVisit registers an new incoming request in the store +func (s *Store) RegisterVisit(id string, visitor Visitor) { + requestID := uuid.New() + logrus.WithFields(logrus.Fields{ + "ClientIP": visitor.IP, + "ID": id, + "RequestID": requestID, + }).Info("New redirect was registered...") + err := s.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(id)) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + data, err := json.Marshal(visitor) + if err != nil { + return errors.Wrap(err, "could not create json") + } + return bucket.Put([]byte(requestID), data) + }) + if err != nil { + logrus.Warningf("could not register visit: %v", err) + } +} + +// GetVisitors returns all the visits of a shorted URL +func (s *Store) GetVisitors(id string) ([]Visitor, error) { + output := []Visitor{} + return output, s.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists([]byte(id)) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return bucket.ForEach(func(k, v []byte) error { + var value Visitor + if err := json.Unmarshal(v, &value); err != nil { + return errors.Wrap(err, "could not unmarshal json") + } + output = append(output, value) + return nil + }) + }) +} + +// GetUserEntries returns all the shorted URL entries of an user +func (s *Store) GetUserEntries(oAuthProvider, oAuthID string) (map[string]Entry, error) { + entries := map[string]Entry{} + return entries, s.db.Update(func(tx *bolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return bucket.ForEach(func(k, v []byte) error { + if bytes.Equal(v, []byte(oAuthProvider+oAuthID)) { + entry, err := s.GetEntryByID(string(k)) + if err != nil { + return errors.Wrap(err, "could not get entry") + } + entries[string(k)] = *entry + } + return nil + }) }) } diff --git a/store/util.go b/store/util.go index b909f13..3087c10 100644 --- a/store/util.go +++ b/store/util.go @@ -15,16 +15,20 @@ import ( ) // createEntryRaw creates a entry with the given key value pair -func (s *Store) createEntryRaw(key, value []byte) error { +func (s *Store) createEntryRaw(key, value, userIdentifier []byte) error { return s.db.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket(s.bucketName) + bucket := tx.Bucket(shortedURLsBucket) if raw := bucket.Get(key); raw != nil { return errors.New("entry already exists") } if err := bucket.Put(key, value); err != nil { return errors.Wrap(err, "could not put data into bucket") } - return nil + uTsURLsBucket, err := tx.CreateBucketIfNotExists(shortedIDsToUserBucket) + if err != nil { + return errors.Wrap(err, "could not create bucket") + } + return uTsURLsBucket.Put(key, userIdentifier) }) } @@ -46,7 +50,7 @@ func (s *Store) createEntry(entry Entry, entryID string) (string, []byte, error) if _, err := mac.Write([]byte(entryID)); err != nil { return "", nil, errors.Wrap(err, "could not write hmac") } - return entryID, mac.Sum(nil), s.createEntryRaw([]byte(entryID), rawEntry) + return entryID, mac.Sum(nil), s.createEntryRaw([]byte(entryID), rawEntry, []byte(entry.OAuthProvider+entry.OAuthID)) } // generateRandomString generates a random string with an predefined length