diff --git a/server/announce.go b/announce/announce.go similarity index 53% rename from server/announce.go rename to announce/announce.go index d671e60..918f911 100644 --- a/server/announce.go +++ b/announce/announce.go @@ -1,4 +1,4 @@ -package server +package announce import ( "fmt" @@ -6,57 +6,64 @@ import ( "net/url" "strconv" "strings" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" ) -func (a *announceData) parseAnnounceData(req *http.Request) (err error) { +func (a *AnnounceData) ParseAnnounceData(req *http.Request) (err error) { query := req.URL.Query() - a.info_hash = ParseInfoHash(query.Get("info_hash")) - if a.info_hash == "" { - err = fmt.Errorf("No info_hash provided.") + + a.RequestContext = requestAppContext{ + dbConn: nil, + Whitelist: false, + } + + a.InfoHash = ParseInfoHash(query.Get("InfoHash")) + if a.InfoHash == "" { + err = fmt.Errorf("No InfoHash provided.") return } if strings.Contains(req.RemoteAddr, ":") { - a.ip = strings.Split(req.RemoteAddr, ":")[0] + a.IP = strings.Split(req.RemoteAddr, ":")[0] } else { - a.ip = query.Get(req.RemoteAddr) + a.IP = query.Get(req.RemoteAddr) } - a.peer_id = query.Get("peer_id") + a.PeerID = query.Get("peer_id") - a.port, err = GetInt(query, "port") + a.Port, err = GetInt(query, "port") if err != nil { return fmt.Errorf("Failed to get port") } - a.downloaded, err = GetInt(query, "downloaded") + a.Downloaded, err = GetInt(query, "downloaded") if err != nil { err = fmt.Errorf("Failed to get downloaded byte count.") return } - a.uploaded, err = GetInt(query, "uploaded") + a.Uploaded, err = GetInt(query, "uploaded") if err != nil { err = fmt.Errorf("Failed to get uploaded byte count.") return } - a.left, err = GetInt(query, "left") + a.Left, err = GetInt(query, "left") if err != nil { err = fmt.Errorf("Failed to get remaining byte count.") return } - a.numwant, err = GetInt(query, "numwant") + a.Numwant, err = GetInt(query, "numwant") if err != nil { - a.numwant = 0 + a.Numwant = 0 } if x := query.Get("compact"); x != "" { - a.compact, err = strconv.ParseBool(x) + a.Compact, err = strconv.ParseBool(x) if err != nil { - a.compact = false + a.Compact = false } } - a.event = query.Get("event") - if a.event == " " || a.event == "" { - a.event = "started" + a.Event = query.Get("event") + if a.Event == " " || a.Event == "" { + a.Event = "started" } - a.requestContext.redisClient = OpenClient() + a.RequestContext.redisClient = r.OpenClient() return } @@ -78,16 +85,16 @@ func GetInt(u url.Values, key string) (ui uint64, err error) { // StartedEventHandler handles whenever a peer sends the STARTED event to the // tracker. -func (a *announceData) StartedEventHandler() (err error) { +func (a *AnnounceData) StartedEventHandler() (err error) { // Called upon announce when a client starts a download or creates a new // torrent on the tracker. Adds a user to incomplete list in redis. err = nil - if !a.infoHashExists() && a.requestContext.whitelist { + if !a.infoHashExists() && a.RequestContext.Whitelist { err = fmt.Errorf("Torrent not authorized for use") return - } else if !a.infoHashExists() && !a.requestContext.whitelist { - // If the info hash isn't in redis and we're not whitelisting, add it + } else if !a.infoHashExists() && !a.RequestContext.Whitelist { + // If the info hash isn't in redis and we're not Whitelisting, add it // to Redis. a.createInfoHashKey() } @@ -95,16 +102,16 @@ func (a *announceData) StartedEventHandler() (err error) { keymember := "" ipport := "" - if !(a.left == 0) { - keymember = fmt.Sprintf("%s:incomplete", a.info_hash) - ipport = fmt.Sprintf("%s:%d", a.ip, a.port) + if !(a.Left == 0) { + keymember = fmt.Sprintf("%s:incomplete", a.InfoHash) + ipport = fmt.Sprintf("%s:%d", a.IP, a.Port) } else { - keymember = fmt.Sprintf("%s:complete", a.info_hash) - ipport = fmt.Sprintf("%s:%d", a.ip, a.port) + keymember = fmt.Sprintf("%s:complete", a.InfoHash) + ipport = fmt.Sprintf("%s:%d", a.IP, a.Port) } - RedisSetKeyVal(a.requestContext.redisClient, keymember, ipport) - if RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { + r.RedisSetKeyVal(a.RequestContext.redisClient, keymember, ipport) + if r.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { fmt.Printf("Adding host %s to %s\n", ipport, keymember) } @@ -116,10 +123,10 @@ func (a *announceData) StartedEventHandler() (err error) { // TODO(ian): This is what happened whenever the torrent client shuts down // gracefully, so we need to call the mysql backend and store the info and // remove the ipport from completed/incomplete redis kvs -func (a *announceData) StoppedEventHandler() { +func (a *AnnounceData) StoppedEventHandler() { if a.infoHashExists() { - a.removeFromKVStorage(a.event) + a.removeFromKVStorage(a.Event) } else { return } @@ -128,7 +135,7 @@ func (a *announceData) StoppedEventHandler() { // CompletedEventHandler Called upon announce when a client finishes a download. Removes the // client from incomplete in redis and places their peer info into // complete. -func (a *announceData) CompletedEventHandler() { +func (a *AnnounceData) CompletedEventHandler() { if !a.infoHashExists() { a.createInfoHashKey() @@ -136,29 +143,29 @@ func (a *announceData) CompletedEventHandler() { a.removeFromKVStorage("incomplete") } - keymember := fmt.Sprintf("%s:complete", a.info_hash) + keymember := fmt.Sprintf("%s:complete", a.InfoHash) // TODO(ian): DRY! - ipport := fmt.Sprintf("%s:%s", a.ip, a.port) - if RedisSetKeyIfNotExists(a.requestContext.redisClient, keymember, ipport) { - fmt.Printf("Adding host %s to %s:complete\n", ipport, a.info_hash) + ipport := fmt.Sprintf("%s:%s", a.IP, a.Port) + if r.RedisSetKeyIfNotExists(a.RequestContext.redisClient, keymember, ipport) { + fmt.Printf("Adding host %s to %s:complete\n", ipport, a.InfoHash) } } -func (a *announceData) removeFromKVStorage(subkey string) { +func (a *AnnounceData) removeFromKVStorage(subkey string) { // Remove the subkey from the kv storage. - ipport := fmt.Sprintf("%s:%d", a.ip, a.port) - keymember := fmt.Sprintf("%s:%s", a.info_hash, subkey) + ipport := fmt.Sprintf("%s:%d", a.IP, a.Port) + keymember := fmt.Sprintf("%s:%s", a.InfoHash, subkey) fmt.Printf("Removing host %s from %v\n", ipport, keymember) - RedisRemoveKeysValue(a.requestContext.redisClient, keymember, ipport) + r.RedisRemoveKeysValue(a.RequestContext.redisClient, keymember, ipport) } -func (a *announceData) infoHashExists() bool { - return RedisGetBoolKeyVal(a.requestContext.redisClient, a.info_hash) +func (a *AnnounceData) infoHashExists() bool { + return r.RedisGetBoolKeyVal(a.InfoHash) } -func (a *announceData) createInfoHashKey() { - CreateNewTorrentKey(a.requestContext.redisClient, a.info_hash) +func (a *AnnounceData) createInfoHashKey() { + r.CreateNewTorrentKey(a.InfoHash) } // ParseInfoHash parses the encoded info hash. Such a simple solution for a diff --git a/server/server_test.go b/announce/announce_test.go similarity index 99% rename from server/server_test.go rename to announce/announce_test.go index ec6e56f..e66cd0f 100644 --- a/server/server_test.go +++ b/announce/announce_test.go @@ -1,4 +1,4 @@ -package server +package announce import ( "fmt" diff --git a/announce/definitions.go b/announce/definitions.go new file mode 100644 index 0000000..c00c8a1 --- /dev/null +++ b/announce/definitions.go @@ -0,0 +1,46 @@ +package announce + +import ( + "github.com/jinzhu/gorm" + "gopkg.in/redis.v3" +) + +const ( + RATIOLESS = iota + SEMIRATIOLESS + NORMALRATIO +) + +type AnnounceData struct { + InfoHash string //20 byte sha1 hash + PeerID string //max len 20 + IP string //optional + Event string // TorrentEvent + + Port uint64 // port number the peer is listening + // on + + Uploaded uint64 // base10 ascii amount uploaded so far + Downloaded uint64 // base10 ascii amount downloaded so + // far + + Left uint64 // # of bytes left to download + // (base 10 ascii) + + Numwant uint64 // Number of peers requested by client. + + Compact bool // Bep23 peer list compression + // decision: True -> compress bep23 + + RequestContext requestAppContext // The request-specific connections +} + +// requestAppContext First of all naming things is the hardest part of +// programming real talk. Second of all, this essentially houses +// request-specific data like db connections and in the future the redisClient. +// Things that should persist only within the duration of a request. +type requestAppContext struct { + dbConn *gorm.DB + redisClient *redis.Client // The redis client connection handler to use. + Whitelist bool +} diff --git a/server/redis.go b/kvStoreInterfaces/redis.go similarity index 73% rename from server/redis.go rename to kvStoreInterfaces/redis.go index a71d009..83a1fe6 100644 --- a/server/redis.go +++ b/kvStoreInterfaces/redis.go @@ -1,9 +1,9 @@ -package server +package kvStoreInterface import ( + "gopkg.in/redis.v3" "bytes" "fmt" - "gopkg.in/redis.v3" "strings" "time" ) @@ -24,16 +24,16 @@ func OpenClient() (client *redis.Client) { } // RedisSetIPMember sets a key as a member of an infohash and sets a timeout. -func RedisSetIPMember(data *announceData) (retval int) { - c := data.requestContext.redisClient +func RedisSetIPMember(infoHash, ipPort string) (retval int) { + c := OpenClient() - keymember := concatenateKeyMember(data.info_hash, "ip") + keymember := concatenateKeyMember(infoHash, "ip") currTime := int64(time.Now().UTC().AddDate(0, 0, 1).Unix()) - ipPort := fmt.Sprintf("%s:%v", createIPPortPair(data), currTime) + key := fmt.Sprintf("%s:%v", ipPort, currTime) - if err := c.SAdd(keymember, ipPort).Err(); err != nil { + if err := c.SAdd(keymember, key).Err(); err != nil { retval = 0 panic("Failed to add key") @@ -61,30 +61,34 @@ func RedisSetKeyVal(c *redis.Client, keymember string, value string) { } // RedisGetKeyVal Lookup a peer in the specified infohash at `key` -func RedisGetKeyVal(data *announceData, key string) []string { +func RedisGetKeyVal(key string) []string { + c := OpenClient() + // RedisGetKeyVal retrieves a value from the Redis store by looking up the - // provided key. If the key does not yet exist, we create the key in the KV + // provided key. If the key does not yet exist, we Create the key in the KV // storage or if the value is empty, we add the current requester to the // list. keymember := concatenateKeyMember(key, "complete") - val, err := data.requestContext.redisClient.SMembers(keymember).Result() + val, err := c.SMembers(keymember).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(data.requestContext.redisClient, keymember) + CreateNewTorrentKey(keymember) } return val } // RedisGetAllPeers fetches all peers from the info_hash at `key` -func RedisGetAllPeers(data *announceData, key string) []string { +func RedisGetAllPeers(key string) []string { + c := OpenClient() + keymember := concatenateKeyMember(key, "complete") - val, err := data.requestContext.redisClient.SRandMemberN(keymember, 30).Result() + val, err := c.SRandMemberN(keymember, 30).Result() if err != nil { // Fail because the key doesn't exist in the KV storage. - CreateNewTorrentKey(data.requestContext.redisClient, keymember) + CreateNewTorrentKey(keymember) } if len(val) == 30 { @@ -93,7 +97,7 @@ func RedisGetAllPeers(data *announceData, key string) []string { keymember = concatenateKeyMember(key, "incomplete") - val2, err := data.requestContext.redisClient.SRandMemberN(keymember, int64(30-len(val))).Result() + val2, err := c.SRandMemberN(keymember, int64(30-len(val))).Result() if err != nil { panic("Failed to get incomplete peers for") } else { @@ -120,15 +124,16 @@ func RedisGetCount(c *redis.Client, info_hash string, member string) (retval int } // RedisGetBoolKeyVal Checks if a `key` exists -func RedisGetBoolKeyVal(client *redis.Client, key string) bool { - ret, _ := client.Exists(key).Result() +func RedisGetBoolKeyVal(key string) bool { + c := OpenClient() + ret, _ := c.Exists(key).Result() return ret } // RedisSetKeyIfNotExists Set a key if it doesn't exist. func RedisSetKeyIfNotExists(c *redis.Client, keymember string, value string) (rv bool) { - rv = RedisGetBoolKeyVal(c, keymember) + rv = RedisGetBoolKeyVal(keymember) if !rv { RedisSetKeyVal(c, keymember, value) } @@ -142,11 +147,12 @@ func RedisRemoveKeysValue(c *redis.Client, key string, value string) { c.SRem(key, value) } -// CreateNewTorrentKey creates a new key. By default, it adds a member +// CreateNewTorrentKey Creates a new key. By default, it adds a member // ":ip". I don't think this ought to ever be generalized, as I just want // Redis to function in one specific way in notorious. -func CreateNewTorrentKey(client *redis.Client, key string) { - client.SAdd(key, "complete", "incomplete") +func CreateNewTorrentKey(key string) { + c := OpenClient() + c.SAdd(key, "complete", "incomplete") } @@ -162,8 +168,8 @@ func concatenateKeyMember(key string, member string) string { return buffer.String() } -// createIPPortPair creates a string formatted ("%s:%s", value.ip, +// CreateIPPortPair Creates a string formatted ("%s:%s", value.ip, // value.port) looking like so: "127.0.0.1:6886" and returns this value. -func createIPPortPair(value *announceData) string { - return fmt.Sprintf("%v:%v", value.ip, value.port) +func createIPPortPair(ip, port string) string { + return fmt.Sprintf("%v:%v", ip, port) } diff --git a/reaper/reaper.go b/reaper/reaper.go index 6e228b3..eaae9bf 100644 --- a/reaper/reaper.go +++ b/reaper/reaper.go @@ -3,7 +3,7 @@ package reaper import ( "fmt" "github.com/GrappigPanda/notorious/database" - "github.com/GrappigPanda/notorious/server" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "gopkg.in/redis.v3" "strconv" "strings" @@ -72,7 +72,7 @@ func StartReapingScheduler(waitTime time.Duration) { go func() { for { // Handle any other cleanup or Notorious-related functions - c := server.OpenClient() + c := r.OpenClient() _, err := c.Ping().Result() if err != nil { panic("No Redis instance detected. If deploying without Docker, install redis-server") @@ -86,7 +86,7 @@ func StartReapingScheduler(waitTime time.Duration) { x, err := db.GetWhitelistedTorrents() for x.Next() { x.Scan(infoHash, name, addedBy, dateAdded) - server.CreateNewTorrentKey(c, *infoHash) + r.CreateNewTorrentKey(*infoHash) } // Start the actual peer reaper. diff --git a/server/announce_response.go b/server/announce_response.go index 221d3d4..d6e86eb 100644 --- a/server/announce_response.go +++ b/server/announce_response.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/GrappigPanda/notorious/bencode" "github.com/GrappigPanda/notorious/database" + . "github.com/GrappigPanda/notorious/announce" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "net" "strconv" "strings" @@ -62,20 +64,20 @@ func CompactAllPeers(ipport []string) []byte { return ret.Bytes() } -func formatResponseData(ips []string, data *announceData) string { +func formatResponseData(ips []string, data *AnnounceData) string { return EncodeResponse(ips, data) } // EncodeResponse groups all of the peer-requested data into a nice bencoded // string that we respond with. -func EncodeResponse(ipport []string, data *announceData) (resp string) { +func EncodeResponse(ipport []string, data *AnnounceData) (resp string) { ret := "" - completeCount := len(RedisGetKeyVal(data, data.info_hash)) - incompleteCount := len(RedisGetKeyVal(data, data.info_hash)) + completeCount := len(r.RedisGetKeyVal(data.InfoHash)) + incompleteCount := len(r.RedisGetKeyVal(data.InfoHash)) ret += bencode.EncodeKV("complete", bencode.EncodeInt(completeCount)) ret += bencode.EncodeKV("incomplete", bencode.EncodeInt(incompleteCount)) - if data.compact || !data.compact { + if data.Compact || !data.Compact { ipstr := string(CompactAllPeers(ipport)) ret += bencode.EncodeKV("peers", ipstr) } else { diff --git a/server/announce_test.go b/server/announce_test.go deleted file mode 100644 index 8520c3d..0000000 --- a/server/announce_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package server - -import ( - "github.com/GrappigPanda/notorious/database" - "testing" -) - -var DBCONN, _ = db.OpenConnection() - -var CONTEXT = requestAppContext{ - dbConn: DBCONN, - redisClient: OpenClient(), - whitelist: true, -} - -var ANNOUNCEDATA = &announceData{ - info_hash: "54321543215432154321", - peer_id: "11111111111111111111", - ip: "127.0.0.1", - event: "STARTED", - port: 6667, - uploaded: 0, - downloaded: 0, - left: 0, - numwant: 30, - compact: true, - requestContext: CONTEXT, -} - -// TestStartedEventHandler tests that with a whitelist being active, we can not -// add a new info_hash to the tracker. -func TestStartedEventHandler(t *testing.T) { - err := ANNOUNCEDATA.StartedEventHandler() - - if err == nil { - t.Fatalf("Failed to TestStartedEventHandler: %v", err) - } -} - -// TestStartedEventHandler tests that with a whitelist being active, we can not -// add a new info_hash to the tracker. -func TestStartedEventHandlerNoWhitelist(t *testing.T) { - announce2 := ANNOUNCEDATA - announce2.requestContext.whitelist = false - err := announce2.StartedEventHandler() - - if err != nil { - t.Fatalf("Failed to TestStartedEventHandler: %v", err) - } -} diff --git a/server/definitions.go b/server/definitions.go deleted file mode 100644 index 4f7e337..0000000 --- a/server/definitions.go +++ /dev/null @@ -1,82 +0,0 @@ -package server - -import ( - "github.com/GrappigPanda/notorious/config" - "github.com/jinzhu/gorm" - "gopkg.in/redis.v3" -) - -const ( - RATIOLESS = iota - SEMIRATIOLESS - NORMALRATIO -) - -type announceData struct { - info_hash string //20 byte sha1 hash - peer_id string //max len 20 - ip string //optional - event string // TorrentEvent - - port uint64 // port number the peer is listening - // on - - uploaded uint64 // base10 ascii amount uploaded so far - downloaded uint64 // base10 ascii amount downloaded so - // far - - left uint64 // # of bytes left to download - // (base 10 ascii) - - numwant uint64 // Number of peers requested by client. - - compact bool // Bep23 peer list compression - // decision: True -> compress bep23 - - requestContext requestAppContext // The request-specific connections -} - -// requestAppContext First of all naming things is the hardest part of -// programming real talk. Second of all, this essentially houses -// request-specific data like db connections and in the future the redisClient. -// Things that should persist only within the duration of a request. -type requestAppContext struct { - dbConn *gorm.DB - redisClient *redis.Client // The redis client connection handler to use. - whitelist bool -} - -type scrapeData struct { - infoHash string -} - -// scrapeResponse is the data associated with a returned scrape. -type scrapeResponse struct { - complete uint64 - downloaded uint64 - incomplete uint64 -} - -// TorrentResponseData models what is sent back to the peer upon a succesful -// info hash lookup. -type TorrentResponseData struct { - interval int - min_interval int - tracker_id string - completed int - incomplete int - peers interface{} -} - -// ANNOUNCE_URL The announce path for the http calls to reach. -var ANNOUNCE_URL = "/announce" - -// TODO(ian): Set this expireTime to a config-loaded value. -// expireTime := 5 * 60 - -// applicationContext houses data necessary for the handler to properly -// function as the application is desired. -type applicationContext struct { - config config.ConfigStruct - trackerLevel int -} diff --git a/server/peerStore/peerstore.go b/server/peerStore/peerstore.go new file mode 100644 index 0000000..5ada357 --- /dev/null +++ b/server/peerStore/peerstore.go @@ -0,0 +1,11 @@ +package peerStore + +type PeerStore interface { + SetKeyIfNotExists(string, string) bool + SetKV(string, string) + RemoveKV(string, string) + KeyExists(string) bool + GetKeyVal(string) []string + GetAllPeers(string) []string + SetIPMember(string, string) int +} diff --git a/server/peerStore/redisStore.go b/server/peerStore/redisStore.go new file mode 100644 index 0000000..93b56f5 --- /dev/null +++ b/server/peerStore/redisStore.go @@ -0,0 +1,41 @@ +package peerStore + +import ( + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" + "gopkg.in/redis.v3" +) + +type RedisStore struct { + client *redis.Client +} + +func (p *RedisStore) SetKeyIfNotExists(key, value string) (retval bool) { + return r.RedisSetKeyIfNotExists(p.client, key, value) +} + +func (p *RedisStore) SetKV(key, value string) { + r.RedisSetKeyVal(p.client, key, value) +} + +func (p *RedisStore) RemoveKV(key, value string) { + // TODO(ian): Refactor this so we don't have to delete a value from a key + if value != "" || value == "" { + r.RedisRemoveKeysValue(p.client, key, value) + } +} + +func (p *RedisStore) KeyExists(key string) (retval bool) { + return r.RedisGetBoolKeyVal(key) +} + +func (p *RedisStore) GetKeyVal(key string) []string { + return r.RedisGetKeyVal(key) +} + +func (p *RedisStore) GetAllPeers(key string) []string { + return r.RedisGetAllPeers(key) +} + +func (p *RedisStore) SetIPMember(infoHash, ipPort string) (retval int) { + return r.RedisSetIPMember(infoHash, ipPort) +} diff --git a/server/redis_test.go b/server/redis_test.go deleted file mode 100644 index fdf2a34..0000000 --- a/server/redis_test.go +++ /dev/null @@ -1,308 +0,0 @@ -package server - -import ( - "testing" -) - -var reqContext = requestAppContext{ - redisClient: OpenClient(), - dbConn: nil, -} - -var DATA = announceData{ - info_hash: "12345123451234512345", - peer_id: "12345123451234512345", - ip: "127.0.0.1", - event: "STARTED", - port: 6767, - uploaded: 1024, - downloaded: 512, - left: 0, - numwant: 30, - compact: true, - requestContext: reqContext, -} - -func TestRedisSetIPMember(t *testing.T) { - ret := RedisSetIPMember(&DATA) - - expectedReturn := 1 - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyVal(t *testing.T) { - RedisSetKeyVal(DATA.requestContext.redisClient, "test:1234", "1024") - - ret, _ := DATA.requestContext.redisClient.SMembers("test:1234").Result() - - expectedReturn := ">1" - - if len(ret) == 0 { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestRedisGetKeyVal(t *testing.T) { - DATA.requestContext.redisClient.SAdd("RedisGetKeyValTest:1024:complete", "1024") - ret := RedisGetKeyVal(&DATA, "RedisGetKeyValTest:1024") - expectedReturn := ">1" - - if len(ret) == 0 { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestRedisGetKeyValNoPreexistKey(t *testing.T) { - DATA.requestContext.redisClient.SAdd("RedisGetKeyValTest:1025", "1024") - ret := RedisGetKeyVal(&DATA, "RedisGetKeyValTest:1025") - expectedReturn := 0 - - if len(ret) != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, len(ret)) - } -} - -func TestCreateIpPortPair(t *testing.T) { - expectedReturn := "127.0.0.1:6767" - ret := createIPPortPair(&DATA) - - if expectedReturn != ret { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestConcatenateKeyMember(t *testing.T) { - expectedReturn := "127.0.0.1:1024" - ret := concatenateKeyMember("127.0.0.1", "1024") - - if expectedReturn != ret { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestCreateNewTorrentKey(t *testing.T) { - CreateNewTorrentKey(DATA.requestContext.redisClient, "testTestCreateNewTorrentKey") - - ret, err := DATA.requestContext.redisClient.Exists("testTestCreateNewTorrentKey").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret != true { - t.Fatalf("CreateNewTorrentKey:complete failed to create") - } - - ret, err = DATA.requestContext.redisClient.SIsMember("testTestCreateNewTorrentKey", "complete").Result() - if ret != true { - t.Fatalf("testTestCreateNewTorrentKey:complete is not a member") - } - - ret, err = DATA.requestContext.redisClient.SIsMember("testTestCreateNewTorrentKey", "incomplete").Result() - if ret != true { - t.Fatalf("testTestCreateNewTorrentKey:incomplete is not a member") - } - -} - -func TestRedisRemoveKeyValues(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisRemoveKeyVal", "Test1") - ret, err := DATA.requestContext.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret != true { - t.Fatalf("Failed in setup of TestRedisRemoveKeyValues to add a key") - } - - RedisRemoveKeysValue(DATA.requestContext.redisClient, "TestRedisRemoveKeyVal", "Test1") - ret, err = DATA.requestContext.redisClient.SIsMember("TestRedisRemoveKeyVal", "Test1").Result() - if err != nil { - t.Fatalf("%v", err) - } - if ret == true { - t.Fatalf("RedisRemoveKeyVal failed to remove the key") - } - -} - -func TestRedisGetBoolKeyVal(t *testing.T) { - RedisSetKeyVal(DATA.requestContext.redisClient, "TestRedisGetBoolKeyVal", "1024") - - expectedReturn := true - ret := RedisGetBoolKeyVal(DATA.requestContext.redisClient, "TestRedisGetBoolKeyVal") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyIfNotExists(t *testing.T) { - expectedReturn := false - ret := RedisSetKeyIfNotExists(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisSetKeyIfNotExistsPreExistingKey(t *testing.T) { - expectedReturn := true - RedisSetKeyVal(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - ret := RedisSetKeyIfNotExists(DATA.requestContext.redisClient, "TestRedisSetKeyIfNotExists", "1024") - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisGetCount(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetCount", "Test") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1235") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1236") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1237") - DATA.requestContext.redisClient.SAdd("TestRedisGetCount:Test", "1238") - - expectedReturn := 4 - ret, err := RedisGetCount(DATA.requestContext.redisClient, "TestRedisGetCount", "Test") - if err != nil { - t.Fatalf("%v", err) - } - - if ret != expectedReturn { - t.Fatalf("Expected %v, got %v", expectedReturn, ret) - } -} - -func TestRedisGetAllPeers(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1235") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1236") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1237") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers:complete", "1238") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers") - x := len(ret) - - if x != 4 { - t.Fatalf("Expected 4 peers, got %v", x) - } -} - -func TestRedisGetAllPeersValGT30(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1206") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1207") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1208") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1210") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1211") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1212") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1213") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1214") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1215") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1200") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1219") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1220") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1223") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1224") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1225") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1226") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1227") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1228") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1229") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1230") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1231") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1232") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1233") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1234") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers1:complete", "1235") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers1") - x := len(ret) - - if x != 30 { - t.Fatalf("Expected 30 peers, got %v", x) - } -} - -func TestRedisGetAllPeersValLT30(t *testing.T) { - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2", "complete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1206") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1207") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1208") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1210") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1211") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1212") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1213") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1214") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:complete", "1215") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2", "incomplete") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1209") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1200") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1201") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1202") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1203") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1204") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1205") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1216") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1217") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1218") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1219") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1220") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1221") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1222") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1223") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1224") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1225") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1226") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1227") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1228") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1229") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1230") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1231") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1232") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1233") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1234") - DATA.requestContext.redisClient.SAdd("TestRedisGetAllPeers2:incomplete", "1235") - - ret := RedisGetAllPeers(&DATA, "TestRedisGetAllPeers2") - x := len(ret) - - if x != 30 { - t.Fatalf("Expected 30 peers, got %v", x) - } -} diff --git a/server/server.go b/server/server.go index 57e31b8..3bb84de 100644 --- a/server/server.go +++ b/server/server.go @@ -4,59 +4,97 @@ import ( "fmt" "github.com/GrappigPanda/notorious/config" "github.com/GrappigPanda/notorious/database" + . "github.com/GrappigPanda/notorious/announce" + "github.com/GrappigPanda/notorious/server/peerStore" + r "github.com/GrappigPanda/notorious/kvStoreInterfaces" "net/http" ) +// applicationContext houses data necessary for the handler to properly +// function as the application is desired. +type applicationContext struct { + config config.ConfigStruct + trackerLevel int + peerStoreClient peerStore.PeerStore +} + +type scrapeData struct { + infoHash string +} + +// scrapeResponse is the data associated with a returned scrape. +type scrapeResponse struct { + complete uint64 + downloaded uint64 + incomplete uint64 +} + +// TorrentResponseData models what is sent back to the peer upon a succesful +// info hash lookup. +type TorrentResponseData struct { + interval int + min_interval int + tracker_id string + completed int + incomplete int + peers interface{} +} + +// ANNOUNCE_URL The announce path for the http calls to reach. +var ANNOUNCE_URL = "/announce" + +// TODO(ian): Set this expireTime to a config-loaded value. +// expireTime := 5 * 60 // FIELDS The fields that we expect from a peer upon info hash lookup var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"} -func worker(data *announceData) []string { - if RedisGetBoolKeyVal(data.requestContext.redisClient, data.info_hash) { - x := RedisGetKeyVal(data, data.info_hash) +func (app *applicationContext) worker(data *AnnounceData) []string { + if app.peerStoreClient.KeyExists(data.InfoHash) { + x := app.peerStoreClient.GetKeyVal(data.InfoHash) - RedisSetIPMember(data) + app.peerStoreClient.SetIPMember(data.InfoHash, fmt.Sprintf("%s:%s", data.IP, data.Port)) return x - } + } else { + r.CreateNewTorrentKey(data.InfoHash) + } - CreateNewTorrentKey(data.requestContext.redisClient, data.info_hash) - return worker(data) + return app.worker(data) } -func (app *applicationContext) handleStatsTracking(data *announceData) { - db.UpdateStats(data.uploaded, data.downloaded) + +func (app *applicationContext) handleStatsTracking(data *AnnounceData) { + db.UpdateStats(data.Uploaded, data.Downloaded) if app.trackerLevel > RATIOLESS { - db.UpdatePeerStats(data.uploaded, data.downloaded, data.ip) + db.UpdatePeerStats(data.Uploaded, data.Downloaded, data.IP) } - if data.event == "completed" { + if data.Event == "completed" { db.UpdateTorrentStats(1, -1) return - } else if data.left == 0 { + } else if data.Left == 0 { // TODO(ian): Don't assume the peer is already in the DB db.UpdateTorrentStats(1, -1) return - } else if data.event == "started" { + } else if data.Event == "started" { db.UpdateTorrentStats(0, 1) } } func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.Request) { - data := new(announceData) - data.requestContext = requestAppContext{ - dbConn: nil, - whitelist: app.config.Whitelist, - } - - err := data.parseAnnounceData(req) + data := new(AnnounceData) + err := data.ParseAnnounceData(req) if err != nil { panic(err) + } - fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port) + data.RequestContext.Whitelist = app.config.Whitelist + + fmt.Printf("Event: %s from host %s on port %v\n", data.Event, data.IP, data.Port) - switch data.event { + switch data.Event { case "started": err := data.StartedEventHandler() if err != nil { @@ -73,9 +111,9 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R panic(fmt.Errorf("We're somehow getting this strange error...")) } - if data.event == "started" || data.event == "completed" { - worker(data) - x := RedisGetAllPeers(data, data.info_hash) + if data.Event == "started" || data.Event == "completed" { + app.worker(data) + x := app.peerStoreClient.GetAllPeers(data.InfoHash) if len(x) > 0 { response := formatResponseData(x, data) @@ -83,7 +121,7 @@ func (app *applicationContext) requestHandler(w http.ResponseWriter, req *http.R } else { failMsg := fmt.Sprintf("No peers for torrent %s\n", - data.info_hash) + data.InfoHash) writeErrorResponse(w, failMsg) } } @@ -98,7 +136,7 @@ func scrapeHandler(w http.ResponseWriter, req *http.Request) { panic(err) } - infoHash := query.Get("info_hash") + infoHash := query.Get("InfoHash") if infoHash == "" { failMsg := fmt.Sprintf("Tracker does not support multiple entire DB scrapes.") writeErrorResponse(w, failMsg) @@ -124,6 +162,7 @@ func RunServer() { app := applicationContext{ config: config.LoadConfig(), trackerLevel: RATIOLESS, + peerStoreClient: new(peerStore.RedisStore), } mux := http.NewServeMux()