diff --git a/cmd/travis-account-sync/main.go b/cmd/travis-account-sync/main.go index f21d156..5c45006 100644 --- a/cmd/travis-account-sync/main.go +++ b/cmd/travis-account-sync/main.go @@ -19,7 +19,11 @@ func main() { if err != nil { log.Fatalf("err=%q", err.Error()) } - accountsync.NewSyncer(cfg).Sync() + syncer, err := accountsync.NewSyncer(cfg) + if err != nil { + log.Fatalf("err=%q", err.Error()) + } + syncer.Sync() } app.Run(os.Args) } diff --git a/config.go b/config.go index 351fbfe..c88ec95 100644 --- a/config.go +++ b/config.go @@ -37,6 +37,11 @@ var ( Value: &cli.StringSlice{"public"}, EnvVar: "TRAVIS_ACCOUNT_SYNC_TYPES", } + SyncCacheSizeFlag = &cli.IntFlag{ + Name: "sync-cache-size", + Value: 64, + EnvVar: "TRAVIS_ACCOUNT_SYNC_CACHE_SIZE", + } Flags = []cli.Flag{ *EncryptionKeyFlag, @@ -45,6 +50,7 @@ var ( *OrganizationsRepositoriesLimitFlag, *RepositoriesStartPageFlag, *SyncTypesFlag, + *SyncCacheSizeFlag, } errPrivateSyncNotSupported = fmt.Errorf("private sync is not supported (yet)!") @@ -57,6 +63,7 @@ type Config struct { OrganizationsRepositoriesLimit int `cfg:"organizations-repositories-limit"` RepositoriesStartPage int `cfg:"repositories-start-page"` SyncTypes []string `cfg:"sync-types"` + SyncCacheSize int `cfg:"sync-cache-size"` } func NewConfig(c *cli.Context) *Config { @@ -67,6 +74,7 @@ func NewConfig(c *cli.Context) *Config { OrganizationsRepositoriesLimit: c.Int("organizations-repositories-limit"), RepositoriesStartPage: c.Int("repositories-start-page"), SyncTypes: c.StringSlice("sync-types"), + SyncCacheSize: c.Int("sync-cache-size"), } } diff --git a/db.go b/db.go new file mode 100644 index 0000000..7951590 --- /dev/null +++ b/db.go @@ -0,0 +1,88 @@ +package accountsync + +import ( + "database/sql" + + "github.com/hashicorp/golang-lru" + "github.com/jmoiron/sqlx" +) + +type DB struct { + *sqlx.DB + + uc *lru.Cache + oc *lru.Cache +} + +func NewDB(databaseURL string, syncCacheSize int) (*DB, error) { + db, err := sqlx.Connect("postgres", databaseURL) + if err != nil { + return nil, err + } + + uc, err := lru.New(syncCacheSize) + if err != nil { + return nil, err + } + + oc, err := lru.New(syncCacheSize) + if err != nil { + return nil, err + } + + return &DB{ + DB: db, + uc: uc, + oc: oc, + }, nil +} + +func (db *DB) FindUserByGithubID(ghUserID int) (*User, error) { + var ( + user *User + ok bool + ) + + u, found := db.uc.Get(ghUserID) + if user, ok = u.(*User); found && ok { + return user, nil + } + + user = &User{} + err := db.Get(user, `SELECT * FROM users WHERE github_id = $1`, ghUserID) + if err == sql.ErrNoRows { + user = nil + err = nil + } + + if user != nil { + db.uc.Add(ghUserID, user) + } + + return user, err +} + +func (db *DB) FindOrgByGithubID(ghOrgID int) (*Organization, error) { + var ( + org *Organization + ok bool + ) + + o, found := db.oc.Get(ghOrgID) + if org, ok = o.(*Organization); found && ok { + return org, nil + } + + org = &Organization{} + err := db.Get(org, `SELECT * FROM organizations WHERE github_id = $1`, ghOrgID) + if err == sql.ErrNoRows { + org = nil + err = nil + } + + if org != nil { + db.oc.Add(ghOrgID, org) + } + + return org, err +} diff --git a/organization_syncer.go b/organization_syncer.go index dd2d461..5afa10c 100644 --- a/organization_syncer.go +++ b/organization_syncer.go @@ -4,11 +4,10 @@ import ( "log" "github.com/google/go-github/github" - "github.com/jmoiron/sqlx" ) type OrganizationSyncer struct { - db *sqlx.DB + db *DB cfg *Config } @@ -19,7 +18,7 @@ type orgSyncContext struct { ghOrgs map[string]*github.Organization } -func NewOrganizationSyncer(db *sqlx.DB, cfg *Config) *OrganizationSyncer { +func NewOrganizationSyncer(db *DB, cfg *Config) *OrganizationSyncer { return &OrganizationSyncer{db: db, cfg: cfg} } diff --git a/owner_repositories_syncer.go b/owner_repositories_syncer.go index d19f8bc..0a2c56d 100644 --- a/owner_repositories_syncer.go +++ b/owner_repositories_syncer.go @@ -5,11 +5,10 @@ import ( "strings" "github.com/google/go-github/github" - "github.com/jmoiron/sqlx" ) type OwnerRepositoriesSyncer struct { - db *sqlx.DB + db *DB cfg *Config } @@ -37,7 +36,7 @@ func (eos *errOrgSync) Error() string { return strings.Join(s, "; ") } -func NewOwnerRepositoriesSyncer(db *sqlx.DB, cfg *Config) *OwnerRepositoriesSyncer { +func NewOwnerRepositoriesSyncer(db *DB, cfg *Config) *OwnerRepositoriesSyncer { return &OwnerRepositoriesSyncer{db: db, cfg: cfg} } @@ -73,15 +72,19 @@ func (ors *OwnerRepositoriesSyncer) Sync(user *User, client *github.Client) erro orgSyncErrors := map[string][]error{} for _, owner := range owners { - rs := NewRepositoriesSyncer(ors.db, ors.cfg) - repoIDs, err := rs.Sync(owner, user, client) - if err != nil { + addErr := func(err error) { hadRepoSyncErr = true key := owner.Key() if _, ok := orgSyncErrors[key]; !ok { orgSyncErrors[key] = []error{} } orgSyncErrors[key] = append(orgSyncErrors[key], err) + } + + rs := NewRepositoriesSyncer(ors.db, ors.cfg) + repoIDs, err := rs.Sync(owner, user, client) + if err != nil { + addErr(err) continue } diff --git a/repositories_syncer.go b/repositories_syncer.go index d48b356..c63d9e0 100644 --- a/repositories_syncer.go +++ b/repositories_syncer.go @@ -7,7 +7,6 @@ import ( "time" "github.com/google/go-github/github" - "github.com/jmoiron/sqlx" ) type repoSyncContext struct { @@ -17,11 +16,11 @@ type repoSyncContext struct { } type RepositoriesSyncer struct { - db *sqlx.DB + db *DB cfg *Config } -func NewRepositoriesSyncer(db *sqlx.DB, cfg *Config) *RepositoriesSyncer { +func NewRepositoriesSyncer(db *DB, cfg *Config) *RepositoriesSyncer { return &RepositoriesSyncer{ db: db, cfg: cfg, @@ -188,7 +187,7 @@ func (rs *RepositoriesSyncer) findRepoOwner(ghRepo *github.Repository, ctx *repo owner := &Owner{} log.Printf("level=debug sync=repository msg=\"finding user\" github_id=%v", *ghRepo.Owner.ID) - user, err := rs.findUserByGithubID(*ghRepo.Owner.ID) + user, err := rs.db.FindUserByGithubID(*ghRepo.Owner.ID) if err != nil { return nil, err } @@ -200,7 +199,7 @@ func (rs *RepositoriesSyncer) findRepoOwner(ghRepo *github.Repository, ctx *repo } log.Printf("level=debug sync=repository msg=\"finding org\" github_id=%v", *ghRepo.Owner.ID) - org, err := rs.findOrgByGithubID(*ghRepo.Owner.ID) + org, err := rs.db.FindOrgByGithubID(*ghRepo.Owner.ID) if err != nil { return nil, err } @@ -296,26 +295,6 @@ func (rs *RepositoriesSyncer) updateRepo(repo *Repository, ctx *repoSyncContext) return repo, err } -func (rs *RepositoriesSyncer) findUserByGithubID(ghUserID int) (*User, error) { - user := &User{} - err := rs.db.Get(user, `SELECT * FROM users WHERE github_id = $1`, ghUserID) - if err == sql.ErrNoRows { - user = nil - err = nil - } - return user, err -} - -func (rs *RepositoriesSyncer) findOrgByGithubID(ghOrgID int) (*Organization, error) { - org := &Organization{} - err := rs.db.Get(org, `SELECT * FROM organizations WHERE github_id = $1`, ghOrgID) - if err == sql.ErrNoRows { - org = nil - err = nil - } - return org, err -} - func (rs *RepositoriesSyncer) createRepoOwner(repo *github.Repository, ctx *repoSyncContext) (*Owner, error) { switch *repo.Owner.Type { case "User": diff --git a/syncer.go b/syncer.go index a8ba239..102c33d 100644 --- a/syncer.go +++ b/syncer.go @@ -7,7 +7,6 @@ import ( "time" "github.com/google/go-github/github" - "github.com/jmoiron/sqlx" "github.com/travis-ci/encrypted-column" "golang.org/x/oauth2" @@ -23,12 +22,19 @@ func (ts *tokenSource) Token() (*oauth2.Token, error) { } type Syncer struct { - db *sqlx.DB + db *DB cfg *Config } -func NewSyncer(cfg *Config) *Syncer { - return &Syncer{cfg: cfg} +func NewSyncer(cfg *Config) (*Syncer, error) { + syncer := &Syncer{cfg: cfg} + log.Println("msg=\"creating database connection\"") + db, err := NewDB(syncer.cfg.DatabaseURL, syncer.cfg.SyncCacheSize) + if err != nil { + return nil, err + } + syncer.db = db + return syncer, nil } func (syncer *Syncer) Sync() { @@ -38,16 +44,9 @@ func (syncer *Syncer) Sync() { log.Fatal("msg=\"missing encryption key\"") } - log.Println("msg=\"connecting to database\"") - db, err := sqlx.Connect("postgres", syncer.cfg.DatabaseURL) - if err != nil { - log.Fatal(err) - } - - syncer.db = db - userInfoSyncer := NewUserInfoSyncer(db, syncer.cfg) - orgSyncer := NewOrganizationSyncer(db, syncer.cfg) - ownerReposSyncer := NewOwnerRepositoriesSyncer(db, syncer.cfg) + userInfoSyncer := NewUserInfoSyncer(syncer.db, syncer.cfg) + orgSyncer := NewOrganizationSyncer(syncer.db, syncer.cfg) + ownerReposSyncer := NewOwnerRepositoriesSyncer(syncer.db, syncer.cfg) ghTokCol, err := encryptedcolumn.NewEncryptedColumn(syncer.cfg.EncryptionKey, true) if err != nil { @@ -68,7 +67,7 @@ func (syncer *Syncer) Sync() { } log.Printf("msg=\"fetching user\" login=%v", githubUsername) - err = db.Get(user, "SELECT * FROM users WHERE login = $1", githubUsername) + err = syncer.db.Get(user, "SELECT * FROM users WHERE login = $1", githubUsername) if err != nil { addErr(err) continue diff --git a/user.go b/user.go index e4d1ace..30f377a 100644 --- a/user.go +++ b/user.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/jmoiron/sqlx" "gopkg.in/yaml.v2" ) @@ -49,7 +48,7 @@ func (user *User) Hydrate() error { return yaml.Unmarshal([]byte(user.GithubScopesYAML.String), &user.GithubScopes) } -func (user *User) HydrateOrganizations(db *sqlx.DB) error { +func (user *User) HydrateOrganizations(db *DB) error { if user.Organizations != nil { return nil } diff --git a/user_info_syncer.go b/user_info_syncer.go index fa8ff59..ce871c9 100644 --- a/user_info_syncer.go +++ b/user_info_syncer.go @@ -32,11 +32,11 @@ type userInfoSyncContext struct { } type UserInfoSyncer struct { - db *sqlx.DB + db *DB cfg *Config } -func NewUserInfoSyncer(db *sqlx.DB, cfg *Config) *UserInfoSyncer { +func NewUserInfoSyncer(db *DB, cfg *Config) *UserInfoSyncer { return &UserInfoSyncer{db: db, cfg: cfg} }