Wrap *sqlx.DB in our own with custom finding funcs

plus add rudimentary query cache for users and orgs

cmd/travis-account-sync/main.go

@@ -19,7 +19,11 @@ func main() {
 		if err != nil {
 			log.Fatalf("err=%q", err.Error())
 		}
-		accountsync.NewSyncer(cfg).Sync()
+		syncer, err := accountsync.NewSyncer(cfg)
+		if err != nil {
+			log.Fatalf("err=%q", err.Error())
+		}
+		syncer.Sync()
 	}
 	app.Run(os.Args)
 }

config.go

@@ -37,6 +37,11 @@ var (
 		Value:  &cli.StringSlice{"public"},
 		EnvVar: "TRAVIS_ACCOUNT_SYNC_TYPES",
 	}
+	SyncCacheSizeFlag = &cli.IntFlag{
+		Name:   "sync-cache-size",
+		Value:  64,
+		EnvVar: "TRAVIS_ACCOUNT_SYNC_CACHE_SIZE",
+	}
 
 	Flags = []cli.Flag{
 		*EncryptionKeyFlag,
@@ -45,6 +50,7 @@ var (
 		*OrganizationsRepositoriesLimitFlag,
 		*RepositoriesStartPageFlag,
 		*SyncTypesFlag,
+		*SyncCacheSizeFlag,
 	}
 
 	errPrivateSyncNotSupported = fmt.Errorf("private sync is not supported (yet)!")
@@ -57,6 +63,7 @@ type Config struct {
 	OrganizationsRepositoriesLimit int      `cfg:"organizations-repositories-limit"`
 	RepositoriesStartPage          int      `cfg:"repositories-start-page"`
 	SyncTypes                      []string `cfg:"sync-types"`
+	SyncCacheSize                  int      `cfg:"sync-cache-size"`
 }
 
 func NewConfig(c *cli.Context) *Config {
@@ -67,6 +74,7 @@ func NewConfig(c *cli.Context) *Config {
 		OrganizationsRepositoriesLimit: c.Int("organizations-repositories-limit"),
 		RepositoriesStartPage:          c.Int("repositories-start-page"),
 		SyncTypes:                      c.StringSlice("sync-types"),
+		SyncCacheSize:                  c.Int("sync-cache-size"),
 	}
 }
 

db.go

@@ -0,0 +1,88 @@
+package accountsync
+
+import (
+	"database/sql"
+
+	"github.com/hashicorp/golang-lru"
+	"github.com/jmoiron/sqlx"
+)
+
+type DB struct {
+	*sqlx.DB
+
+	uc *lru.Cache
+	oc *lru.Cache
+}
+
+func NewDB(databaseURL string, syncCacheSize int) (*DB, error) {
+	db, err := sqlx.Connect("postgres", databaseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	uc, err := lru.New(syncCacheSize)
+	if err != nil {
+		return nil, err
+	}
+
+	oc, err := lru.New(syncCacheSize)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DB{
+		DB: db,
+		uc: uc,
+		oc: oc,
+	}, nil
+}
+
+func (db *DB) FindUserByGithubID(ghUserID int) (*User, error) {
+	var (
+		user *User
+		ok   bool
+	)
+
+	u, found := db.uc.Get(ghUserID)
+	if user, ok = u.(*User); found && ok {
+		return user, nil
+	}
+
+	user = &User{}
+	err := db.Get(user, `SELECT * FROM users WHERE github_id = $1`, ghUserID)
+	if err == sql.ErrNoRows {
+		user = nil
+		err = nil
+	}
+
+	if user != nil {
+		db.uc.Add(ghUserID, user)
+	}
+
+	return user, err
+}
+
+func (db *DB) FindOrgByGithubID(ghOrgID int) (*Organization, error) {
+	var (
+		org *Organization
+		ok  bool
+	)
+
+	o, found := db.oc.Get(ghOrgID)
+	if org, ok = o.(*Organization); found && ok {
+		return org, nil
+	}
+
+	org = &Organization{}
+	err := db.Get(org, `SELECT * FROM organizations WHERE github_id = $1`, ghOrgID)
+	if err == sql.ErrNoRows {
+		org = nil
+		err = nil
+	}
+
+	if org != nil {
+		db.oc.Add(ghOrgID, org)
+	}
+
+	return org, err
+}

organization_syncer.go

@@ -4,11 +4,10 @@ import (
 	"log"
 
 	"github.com/google/go-github/github"
-	"github.com/jmoiron/sqlx"
 )
 
 type OrganizationSyncer struct {
-	db  *sqlx.DB
+	db  *DB
 	cfg *Config
 }
 
@@ -19,7 +18,7 @@ type orgSyncContext struct {
 	ghOrgs  map[string]*github.Organization
 }
 
-func NewOrganizationSyncer(db *sqlx.DB, cfg *Config) *OrganizationSyncer {
+func NewOrganizationSyncer(db *DB, cfg *Config) *OrganizationSyncer {
 	return &OrganizationSyncer{db: db, cfg: cfg}
 }
 

owner_repositories_syncer.go

@@ -5,11 +5,10 @@ import (
 	"strings"
 
 	"github.com/google/go-github/github"
-	"github.com/jmoiron/sqlx"
 )
 
 type OwnerRepositoriesSyncer struct {
-	db  *sqlx.DB
+	db  *DB
 	cfg *Config
 }
 
@@ -37,7 +36,7 @@ func (eos *errOrgSync) Error() string {
 	return strings.Join(s, "; ")
 }
 
-func NewOwnerRepositoriesSyncer(db *sqlx.DB, cfg *Config) *OwnerRepositoriesSyncer {
+func NewOwnerRepositoriesSyncer(db *DB, cfg *Config) *OwnerRepositoriesSyncer {
 	return &OwnerRepositoriesSyncer{db: db, cfg: cfg}
 }
 
@@ -73,15 +72,19 @@ func (ors *OwnerRepositoriesSyncer) Sync(user *User, client *github.Client) erro
 	orgSyncErrors := map[string][]error{}
 
 	for _, owner := range owners {
-		rs := NewRepositoriesSyncer(ors.db, ors.cfg)
-		repoIDs, err := rs.Sync(owner, user, client)
-		if err != nil {
+		addErr := func(err error) {
 			hadRepoSyncErr = true
 			key := owner.Key()
 			if _, ok := orgSyncErrors[key]; !ok {
 				orgSyncErrors[key] = []error{}
 			}
 			orgSyncErrors[key] = append(orgSyncErrors[key], err)
+		}
+
+		rs := NewRepositoriesSyncer(ors.db, ors.cfg)
+		repoIDs, err := rs.Sync(owner, user, client)
+		if err != nil {
+			addErr(err)
 			continue
 		}
 

repositories_syncer.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/google/go-github/github"
-	"github.com/jmoiron/sqlx"
 )
 
 type repoSyncContext struct {
@@ -17,11 +16,11 @@ type repoSyncContext struct {
 }
 
 type RepositoriesSyncer struct {
-	db  *sqlx.DB
+	db  *DB
 	cfg *Config
 }
 
-func NewRepositoriesSyncer(db *sqlx.DB, cfg *Config) *RepositoriesSyncer {
+func NewRepositoriesSyncer(db *DB, cfg *Config) *RepositoriesSyncer {
 	return &RepositoriesSyncer{
 		db:  db,
 		cfg: cfg,
@@ -188,7 +187,7 @@ func (rs *RepositoriesSyncer) findRepoOwner(ghRepo *github.Repository, ctx *repo
 	owner := &Owner{}
 
 	log.Printf("level=debug sync=repository msg=\"finding user\" github_id=%v", *ghRepo.Owner.ID)
-	user, err := rs.findUserByGithubID(*ghRepo.Owner.ID)
+	user, err := rs.db.FindUserByGithubID(*ghRepo.Owner.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -200,7 +199,7 @@ func (rs *RepositoriesSyncer) findRepoOwner(ghRepo *github.Repository, ctx *repo
 	}
 
 	log.Printf("level=debug sync=repository msg=\"finding org\" github_id=%v", *ghRepo.Owner.ID)
-	org, err := rs.findOrgByGithubID(*ghRepo.Owner.ID)
+	org, err := rs.db.FindOrgByGithubID(*ghRepo.Owner.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -296,26 +295,6 @@ func (rs *RepositoriesSyncer) updateRepo(repo *Repository, ctx *repoSyncContext)
 	return repo, err
 }
 
-func (rs *RepositoriesSyncer) findUserByGithubID(ghUserID int) (*User, error) {
-	user := &User{}
-	err := rs.db.Get(user, `SELECT * FROM users WHERE github_id = $1`, ghUserID)
-	if err == sql.ErrNoRows {
-		user = nil
-		err = nil
-	}
-	return user, err
-}
-
-func (rs *RepositoriesSyncer) findOrgByGithubID(ghOrgID int) (*Organization, error) {
-	org := &Organization{}
-	err := rs.db.Get(org, `SELECT * FROM organizations WHERE github_id = $1`, ghOrgID)
-	if err == sql.ErrNoRows {
-		org = nil
-		err = nil
-	}
-	return org, err
-}
-
 func (rs *RepositoriesSyncer) createRepoOwner(repo *github.Repository, ctx *repoSyncContext) (*Owner, error) {
 	switch *repo.Owner.Type {
 	case "User":

syncer.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/google/go-github/github"
-	"github.com/jmoiron/sqlx"
 	"github.com/travis-ci/encrypted-column"
 	"golang.org/x/oauth2"
 
@@ -23,12 +22,19 @@ func (ts *tokenSource) Token() (*oauth2.Token, error) {
 }
 
 type Syncer struct {
-	db  *sqlx.DB
+	db  *DB
 	cfg *Config
 }
 
-func NewSyncer(cfg *Config) *Syncer {
-	return &Syncer{cfg: cfg}
+func NewSyncer(cfg *Config) (*Syncer, error) {
+	syncer := &Syncer{cfg: cfg}
+	log.Println("msg=\"creating database connection\"")
+	db, err := NewDB(syncer.cfg.DatabaseURL, syncer.cfg.SyncCacheSize)
+	if err != nil {
+		return nil, err
+	}
+	syncer.db = db
+	return syncer, nil
 }
 
 func (syncer *Syncer) Sync() {
@@ -38,16 +44,9 @@ func (syncer *Syncer) Sync() {
 		log.Fatal("msg=\"missing encryption key\"")
 	}
 
-	log.Println("msg=\"connecting to database\"")
-	db, err := sqlx.Connect("postgres", syncer.cfg.DatabaseURL)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	syncer.db = db
-	userInfoSyncer := NewUserInfoSyncer(db, syncer.cfg)
-	orgSyncer := NewOrganizationSyncer(db, syncer.cfg)
-	ownerReposSyncer := NewOwnerRepositoriesSyncer(db, syncer.cfg)
+	userInfoSyncer := NewUserInfoSyncer(syncer.db, syncer.cfg)
+	orgSyncer := NewOrganizationSyncer(syncer.db, syncer.cfg)
+	ownerReposSyncer := NewOwnerRepositoriesSyncer(syncer.db, syncer.cfg)
 
 	ghTokCol, err := encryptedcolumn.NewEncryptedColumn(syncer.cfg.EncryptionKey, true)
 	if err != nil {
@@ -68,7 +67,7 @@ func (syncer *Syncer) Sync() {
 		}
 
 		log.Printf("msg=\"fetching user\" login=%v", githubUsername)
-		err = db.Get(user, "SELECT * FROM users WHERE login = $1", githubUsername)
+		err = syncer.db.Get(user, "SELECT * FROM users WHERE login = $1", githubUsername)
 		if err != nil {
 			addErr(err)
 			continue

user.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/jmoiron/sqlx"
 	"gopkg.in/yaml.v2"
 )
 
@@ -49,7 +48,7 @@ func (user *User) Hydrate() error {
 	return yaml.Unmarshal([]byte(user.GithubScopesYAML.String), &user.GithubScopes)
 }
 
-func (user *User) HydrateOrganizations(db *sqlx.DB) error {
+func (user *User) HydrateOrganizations(db *DB) error {
 	if user.Organizations != nil {
 		return nil
 	}

user_info_syncer.go

@@ -32,11 +32,11 @@ type userInfoSyncContext struct {
 }
 
 type UserInfoSyncer struct {
-	db  *sqlx.DB
+	db  *DB
 	cfg *Config
 }
 
-func NewUserInfoSyncer(db *sqlx.DB, cfg *Config) *UserInfoSyncer {
+func NewUserInfoSyncer(db *DB, cfg *Config) *UserInfoSyncer {
 	return &UserInfoSyncer{db: db, cfg: cfg}
 }