Skip to content

Commit

Permalink
Update Github storing algorithm
Browse files Browse the repository at this point in the history
  • Loading branch information
@GODrums
GODrums committed Sep 18, 2024
1 parent 0f893bb commit c5b1e00
Show file tree
Hide file tree
Showing 2 changed files with 175 additions and 91 deletions.
54 changes: 39 additions & 15 deletions ...cation-server/src/main/java/de/tum/in/www1/hephaestus/leaderboard/LeaderboardService.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
package de.tum.in.www1.hephaestus.leaderboard;

import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import de.tum.in.www1.hephaestus.codereview.pullrequest.PullRequest;
Expand All @@ -22,6 +26,9 @@ public class LeaderboardService {

private final UserService userService;

@Value("${monitoring.timeframe}")
private int timeframe;

public LeaderboardService(UserService userService) {
this.userService = userService;
}
Expand All @@ -36,28 +43,45 @@ public List<LeaderboardEntry> createLeaderboard() {
if (user.getType() != UserType.USER) {
return null;
}
int comments = user.getIssueComments().size();
AtomicInteger comments = new AtomicInteger(
user.getIssueComments().size() + user.getReviewComments().size());
AtomicInteger changesRequested = new AtomicInteger(0);
AtomicInteger changesApproved = new AtomicInteger(0);
AtomicInteger score = new AtomicInteger(0);
user.getReviews().stream().forEach(review -> {
switch (review.getState()) {
case CHANGES_REQUESTED:
changesRequested.incrementAndGet();
break;
case APPROVED:
changesApproved.incrementAndGet();
break;
default:
break;
}
score.addAndGet(calculateScore(review.getPullRequest()));
});
OffsetDateTime cutOffTime = new Date(System.currentTimeMillis() - 1000 * 60 * 60 * 24 * timeframe)
.toInstant().atOffset(ZoneOffset.UTC);
user.getReviews().stream()
.filter(review -> (review.getCreatedAt() != null && review.getCreatedAt().isAfter(cutOffTime))
|| (review.getUpdatedAt() != null && review.getUpdatedAt().isAfter(cutOffTime)))
.forEach(review -> {
if (user.getLogin().equals("SimonEntholzer")) {
logger.info("State: " + review.getState() + review.toString());
}
if (review.getPullRequest().getAuthor().getLogin().equals(user.getLogin())) {
return;
}
switch (review.getState()) {
case CHANGES_REQUESTED:
changesRequested.incrementAndGet();
break;
case APPROVED:
changesApproved.incrementAndGet();
break;
default:
comments.incrementAndGet();
break;
}
score.addAndGet(calculateScore(review.getPullRequest()));
});
logger.info("User " + user.getLogin() + " has " + user.getReviews().size() + " reviews and " + comments
+ " comments" + " and " + changesRequested + " changes requested and " + changesApproved
+ " approved");
return new LeaderboardEntry(user.getLogin(), user.getAvatarUrl(), user.getName(), user.getType(),
score.get(),
0, // preliminary rank
changesRequested.get(),
changesApproved.get(), comments);
changesApproved.get(),
comments.get());
}).filter(Objects::nonNull).collect(Collectors.toCollection(ArrayList::new));

// update ranks by score
Expand Down
212 changes: 136 additions & 76 deletions ...ation-server/src/main/java/de/tum/in/www1/hephaestus/scheduler/GitHubDataSyncService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,21 @@
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

import org.kohsuke.github.GHDirection;
import org.kohsuke.github.GHObject;
import org.kohsuke.github.GHPullRequest;
import org.kohsuke.github.GHPullRequestReviewComment;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GitHub;
import org.kohsuke.github.GitHubBuilder;
import org.kohsuke.github.PagedIterator;
import org.kohsuke.github.GHPullRequestQueryBuilder.Sort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
Expand Down Expand Up @@ -68,6 +73,9 @@ public class GitHubDataSyncService {
private final PullRequestReviewCommentConverter reviewCommentConverter;
private final UserConverter userConverter;

private Set<User> users = new HashSet<>();
private Set<PullRequestReview> reviews = new HashSet<>();

public GitHubDataSyncService(RepositoryRepository repositoryRepository, PullRequestRepository pullRequestRepository,
PullRequestReviewRepository prReviewRepository,
IssueCommentRepository commentRepository, PullRequestReviewCommentRepository reviewCommentRepository,
Expand Down Expand Up @@ -151,92 +159,112 @@ public Repository fetchRepository(String nameWithOwner) throws IOException {
Set<PullRequest> prs = getPullRequestsFromGHRepository(ghRepo, repository);
logger.info("Found " + prs.size() + " PRs");
repository.setPullRequests(prs);
pullRequestRepository.saveAll(prs);

pullRequestRepository.saveAll(prs);
userRepository.saveAll(users);
repositoryRepository.save(repository);
prReviewRepository.saveAll(reviews);
return repository;
}

private Set<PullRequest> getPullRequestsFromGHRepository(GHRepository ghRepo, Repository repository)
throws IOException {
// Retrieve PRs in pages of 10
return ghRepo.queryPullRequests().list().withPageSize(20).toList().stream()
.takeWhile(pr -> isResourceRecent(pr)).map(pr -> {
PullRequest pullRequest = pullRequestRepository.save(pullRequestConverter.convert(pr));
pullRequest.setRepository(repository);
try {
Collection<IssueComment> comments = getCommentsFromGHPullRequest(pr, pullRequest);
comments = commentRepository.saveAll(comments);
for (IssueComment c : comments) {
pullRequest.addComment(c);
}
} catch (IOException e) {
logger.error("Error while fetching PR comments!");
pullRequest.setComments(new HashSet<>());
}
try {
User prAuthor = getUserFromGHUser(pr.getUser());
prAuthor.addPullRequest(pullRequest);
pullRequest.setAuthor(prAuthor);
} catch (IOException e) {
logger.error("Error while fetching PR author!");
pullRequest.setAuthor(null);
PagedIterator<GHPullRequest> pullRequests = ghRepo.queryPullRequests()
// .state(GHIssueState.ALL)
.sort(Sort.UPDATED)
.direction(GHDirection.DESC)
.list().withPageSize(100).iterator();
Set<PullRequest> prs = new HashSet<>();
while (pullRequests.hasNext()) {
List<GHPullRequest> nextPage = pullRequests.nextPage();
if (!isResourceRecent(nextPage.get(0))) {
break;
}
logger.info("Fetched " + nextPage.size() + " PRs");
prs.addAll(nextPage.stream().map(pr -> {
if (!isResourceRecent(pr)) {
return null;
}
PullRequest pullRequest = pullRequestRepository.save(pullRequestConverter.convert(pr));
pullRequest.setRepository(repository);
try {
Collection<IssueComment> comments = getCommentsFromGHPullRequest(pr, pullRequest);
comments = commentRepository.saveAll(comments);
for (IssueComment c : comments) {
pullRequest.addComment(c);
}
} catch (IOException e) {
logger.error("Error while fetching PR comments!");
pullRequest.setComments(new HashSet<>());
}
try {
User prAuthor = getUserFromGHUser(pr.getUser());
prAuthor.addPullRequest(pullRequest);
pullRequest.setAuthor(prAuthor);
} catch (IOException e) {
// logger.error("Error while fetching PR author!");
pullRequest.setAuthor(null);
}

try {
Collection<PullRequestReview> reviews = pr.listReviews().toList().stream()
.takeWhile(prr -> isResourceRecent(prr)).map(review -> {
PullRequestReview prReview = prReviewRepository
.save(reviewConverter.convert(review));
try {
User reviewAuthor = getUserFromGHUser(review.getUser());
reviewAuthor.addReview(prReview);
prReview.setAuthor(reviewAuthor);
} catch (IOException e) {
logger.error("Error while fetching review owner!");
}
prReview.setPullRequest(pullRequest);
return prReview;
}).collect(Collectors.toSet());
reviews = prReviewRepository.saveAll(reviews);
for (PullRequestReview prReview : reviews) {
pullRequest.addReview(prReview);
try {
Set<PullRequestReview> newReviews = pr.listReviews().toList().stream().map(review -> {
PullRequestReview prReview = prReviewRepository
.save(reviewConverter.convert(review));
try {
User reviewAuthor = getUserFromGHUser(review.getUser());
reviewAuthor.addReview(prReview);
prReview.setAuthor(reviewAuthor);
} catch (IOException e) {
// Dont mind this error as it occurs only for bots
// logger.error("Error while fetching review owner!");
}
} catch (IOException e) {
logger.error("Error while fetching PR reviews!");
pullRequest.setReviews(new HashSet<>());
prReview.setPullRequest(pullRequest);
return prReview;
}).collect(Collectors.toSet());
for (PullRequestReview prReview : newReviews) {
pullRequest.addReview(prReview);
reviews.add(prReview);
}
logger.info("Found " + newReviews.size() + " reviews for PR " + pullRequest.getId());
} catch (IOException e) {
logger.error("Error while fetching PR reviews!");
pullRequest.setReviews(new HashSet<>());
}

try {
pr.listReviewComments().withPageSize(20).toList().stream().takeWhile(rc -> isResourceRecent(rc))
.forEach(c -> handleSinglePullRequestReviewComment(c));
} catch (IOException e) {
logger.error("Error while fetching PR review comments!");
}
try {
List<PullRequestReviewComment> prrComments = pr.listReviewComments().withPageSize(100).toList()
.stream()
.takeWhile(rc -> isResourceRecent(rc))
.map(c -> handleSinglePullRequestReviewComment(c)).filter(Objects::nonNull).toList();
reviewCommentRepository.saveAll(prrComments);
} catch (IOException e) {
logger.error("Error while fetching PR review comments!");
}

return pullRequest;
}).collect(Collectors.toSet());
return pullRequest;
}).filter(Objects::nonNull).collect(Collectors.toSet()));
}
return prs;
}

private void handleSinglePullRequestReviewComment(GHPullRequestReviewComment comment) {
PullRequestReviewComment c = reviewCommentRepository.save(reviewCommentConverter.convert(comment));
private PullRequestReviewComment handleSinglePullRequestReviewComment(GHPullRequestReviewComment comment) {
PullRequestReviewComment prrc = reviewCommentRepository.save(reviewCommentConverter.convert(comment));

Optional<PullRequestReview> review = prReviewRepository
.findByIdWithEagerComments(comment.getPullRequestReviewId());
if (review.isPresent()) {
PullRequestReview prReview = review.get();
c.setReview(prReview);
User commentAuthor;
try {
commentAuthor = getUserFromGHUser(comment.getUser());
commentAuthor.addReviewComment(c);
} catch (IOException e) {
logger.error("Error while fetching author!");
commentAuthor = null;
}
c.setAuthor(commentAuthor);
prReview.addComment(c);
PullRequestReview prReview = getPRRFromReviewId(comment.getPullRequestReviewId());
prrc.setReview(prReview);
User commentAuthor;
try {
commentAuthor = getUserFromGHUser(comment.getUser());
commentAuthor.addReviewComment(prrc);
} catch (IOException e) {
// Dont mind this error as it occurs only for bots
// logger.error("Error while fetching author!" + comment.getPullRequestUrl() + "
// " + comment.getCreatedAt());
commentAuthor = null;
}
prrc.setAuthor(commentAuthor);
prReview.addComment(prrc);
return prrc;
}

/**
Expand All @@ -250,7 +278,7 @@ private void handleSinglePullRequestReviewComment(GHPullRequestReviewComment com
@Transactional
private Set<IssueComment> getCommentsFromGHPullRequest(GHPullRequest pr, PullRequest pullRequest)
throws IOException {
return pr.queryComments().list().withPageSize(20).toList().stream()
return pr.queryComments().list().withPageSize(100).toList().stream()
.map(comment -> {
IssueComment c = commentRepository.save(commentConverter.convert(comment));
c.setPullRequest(pullRequest);
Expand All @@ -259,7 +287,9 @@ private Set<IssueComment> getCommentsFromGHPullRequest(GHPullRequest pr, PullReq
commentAuthor = getUserFromGHUser(comment.getUser());
commentAuthor.addIssueComment(c);
} catch (IOException e) {
logger.error("Error while fetching author!");
// Dont mind this error as it occurs only for bots
// logger.error("Error while fetching author!" + comment.getHtmlUrl() + " " +
// comment.getCreatedAt());
commentAuthor = null;
}
c.setAuthor(commentAuthor);
Expand All @@ -268,11 +298,31 @@ private Set<IssueComment> getCommentsFromGHPullRequest(GHPullRequest pr, PullReq
}

private User getUserFromGHUser(org.kohsuke.github.GHUser user) {
User ghUser = userRepository.findUserEagerly(user.getLogin()).orElse(null);
if (ghUser == null) {
ghUser = userRepository.save(userConverter.convert(user));
Optional<User> ghUser = users.stream().filter(u -> u.getLogin().equals(user.getLogin())).findFirst();
if (ghUser.isPresent()) {
return ghUser.get();
}
ghUser = userRepository.findUserEagerly(user.getLogin());
if (ghUser.isPresent()) {
User u = ghUser.get();
if (!users.contains(u)) {
users.add(u);
}
return u;
}
return ghUser;
logger.info("Creating user " + user.getLogin());
User u = userRepository.save(userConverter.convert(user));
users.add(u);
return u;
}

private PullRequestReview getPRRFromReviewId(Long reviewId) {
Optional<PullRequestReview> prReview = reviews.stream().filter(prr -> prr.getId().equals(reviewId)).findFirst();
if (prReview.isPresent()) {
return prReview.get();
}
logger.error("Cannot find PRR with ID " + reviewId);
return null;
}

/**
Expand All @@ -282,8 +332,18 @@ private User getUserFromGHUser(org.kohsuke.github.GHUser user) {
* @return
*/
private boolean isResourceRecent(GHObject obj) {
if (obj.getClass().equals(GHPullRequest.class)) {
try {
logger.info("Checking if PR is recent: " + obj.getUpdatedAt() + ", "
+ obj.getCreatedAt());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
try {
return obj.getCreatedAt().after(new Date(System.currentTimeMillis() - 1000 * 60 * 60 * 24 * timeframe));
return obj.getUpdatedAt() != null
&& obj.getUpdatedAt().after(new Date(System.currentTimeMillis() - 1000 * 60 * 60 * 24 * timeframe));
} catch (IOException e) {
logger.error("Error while fetching createdAt! Resource ID: " + obj.getId());
return false;
Expand Down

0 comments on commit c5b1e00

Please sign in to comment.