Reenable archiving old stats data after updating query selectStoriesT…

…oArchive

archive.go

@@ -92,17 +92,20 @@ func (app app) generateStatsDataJSON(ctx context.Context, storyID int) ([]byte,
 }
 
 func (app app) archiveOldStatsData(ctx context.Context) error {
+	app.logger.Info("Looking for old stories to archive")
 	app.logger.Debug("selectStoriesToArchive")
 	storyIDs, err := app.ndb.selectStoriesToArchive(ctx)
 	if err != nil {
 		return errors.Wrap(err, "selectStoriesToArchive")
 	}
-	app.logger.Debug("Finished selectStoriesToArchive", "nStories", len(storyIDs))
+	app.logger.Debug("Finished selectStoriesToArchive")
 
 	if len(storyIDs) == 0 {
 		return nil // Nothing to archive
 	}
 
+	app.logger.Info("Found old stories to archive", "nStories", len(storyIDs))
+
 	sc, err := NewStorageClient()
 	if err != nil {
 		return errors.Wrap(err, "create storage client")

database.go

@@ -390,9 +390,10 @@ func openNewsDatabase(sqliteDataDir string) (newsDatabase, error) {
 			SELECT DISTINCT id
 			FROM dataset
 			join stories using (id)
-			WHERE submissionTime <= unixepoch() - 28*24*60*60
+			WHERE 
+			sampleTime <= unixepoch() - 14*24*60*60
 			and archived = 0
-			limit 100
+			limit 50
 		`
 		ndb.selectStoriesToArchiveStatement, err = ndb.db.Prepare(sql)
 		if err != nil {
@@ -500,7 +501,7 @@ func (ndb newsDatabase) selectStoriesToArchive(ctx context.Context) ([]int, erro
 
 	rows, err := ndb.selectStoriesToArchiveStatement.QueryContext(ctx)
 	if err != nil {
-		return storyIDs, errors.Wrap(err, "selectStoriesToArchiveStatement.Query")
+		return storyIDs, errors.Wrap(err, "selectStoriesToArchiveStatement.QueryContext")
 	}
 
 	for rows.Next() {

main.go

@@ -157,13 +157,12 @@ func (app app) mainLoop(ctx context.Context) {
 				logger.Error("crawlAndPostprocess", err)
 			}
 
-			// logger.Info("Archiving old stats pages data")
-			// err := app.archiveOldStatsData(ctx)
-			// if err != nil {
-			// 	logger.Error("archiveOldStatsData", err)
-			// }
+			err := app.archiveOldStatsData(ctx)
+			if err != nil {
+				app.logger.Error("archiveOldStatsData", err)
+			}
 
-			logger.Debug("Scheduling tick at next minute mark", "seconds", 60-time.Now().Unix()%60)
+			app.logger.Debug("Finished crawl and postprocess")
 
 		case <-ctx.Done():
 			return