diff --git a/src/java/main/br/ufpe/cin/groundhog/License.java b/src/java/main/br/ufpe/cin/groundhog/License.java index 563993b..62b239e 100644 --- a/src/java/main/br/ufpe/cin/groundhog/License.java +++ b/src/java/main/br/ufpe/cin/groundhog/License.java @@ -25,4 +25,8 @@ public String getName() { public String getEntireContent() { return entireContent; } + + public String toString() { + return name; + } } diff --git a/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java b/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java index 8cc098c..c415782 100644 --- a/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java +++ b/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java @@ -48,6 +48,7 @@ public File checkoutToDate(String project, File repositoryFolder, final Date dat FileUtil.getInstance().copyDirectory(repositoryFolder, projectFolder); + logger.info("Checking out to the specific date.."); this.gitClient.checkout(projectFolder, date); return projectFolder; } catch (EmptyProjectAtDateException e) { diff --git a/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java b/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java index 4ec46e9..be02b37 100644 --- a/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java @@ -42,7 +42,7 @@ public File downloadProject(Project project) { this.gitClient.clone(cloneUrl, projectFolder); return projectFolder; } catch (Exception e) { - String error = String.format("Unable to download %s (%s) project", project.getName(), project.getURL()); + String error = String.format("Unable to download %s (%s) project", project.getName(), project.getScmURL()); logger.error(error); throw new DownloadExecption(error); } diff --git a/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java b/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java index 426acbb..ca6305d 100644 --- a/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java +++ b/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java @@ -9,7 +9,6 @@ import java.util.concurrent.Future; import br.ufpe.cin.groundhog.Project; -import br.ufpe.cin.groundhog.main.JsonInput; /** * An abstract class that defines the forge crawl functionality. @@ -26,7 +25,7 @@ public abstract class ForgeCrawler { * Constructs a new ForgeCrawler with a given destinationFolder. */ protected ForgeCrawler() { - this.ex = Executors.newFixedThreadPool(JsonInput.getMaxThreads()); + this.ex = Executors.newCachedThreadPool(); } /** diff --git a/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java b/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java index 2660cf9..98343e1 100644 --- a/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java +++ b/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java @@ -65,8 +65,7 @@ private License extractLicense(String content) { for (String license : Licenses.names()) { if (content.contains(license)) { - logger.info(String.format("License found! %s uses %s license.", - root.getName(), license)); + logger.info(String.format("License found! %s uses %s license.", root.getName(), license)); return new License(license); } } @@ -75,8 +74,7 @@ private License extractLicense(String content) { private boolean containsLicenseWord(String content) { - for (String licenseKeyword : Lists.newArrayList("license", "copyright", - "permission")) { + for (String licenseKeyword : Lists.newArrayList("license", "copyright", "permission")) { if (content.toLowerCase().contains(licenseKeyword)) { return true; } diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 5746470..72dab0e 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -355,7 +355,7 @@ public List getAllForgeProjects(int start, int limit) throws SearchExce int totalRepositories = 0; while(totalRepositories < limit || limit < 0){ - String searchUrl = ROOT + String.format("/repositories?since=%s&language=java&%s", since, this.oauthToken); + String searchUrl = String.format("%s/repositories?since=%s&language=java&%s", ROOT, since, this.oauthToken); String jsonString = requests.get(searchUrl); JsonArray jsonArray = gson.fromJson(jsonString, JsonElement.class).getAsJsonArray(); @@ -363,22 +363,26 @@ public List getAllForgeProjects(int start, int limit) throws SearchExce (totalRepositories + i < limit || limit < 0); i++) { String repoName = jsonArray.get(i).getAsJsonObject().get("name").getAsString(); - String searchUrlLegacy = String.format("%s/legacy/repos/search/%s?language=java&%s", ROOT, repoName, this.oauthToken); + String searchUrlLegacy = String.format("%s/legacy/repos/search/%s?language=java%s", ROOT, repoName, this.oauthToken); String jsonLegacy = requests.get(searchUrlLegacy); JsonObject jsonObject = gson.fromJson(jsonLegacy, JsonElement.class).getAsJsonObject(); JsonArray jsonArrayLegacy = jsonObject.get("repositories").getAsJsonArray(); - if( jsonArrayLegacy.size() == 0 ) continue; // not a java project + if( jsonArrayLegacy.size() > 0) { - String element = jsonArrayLegacy.get(0).toString(); // results are sorted by best match - Project p = gson.fromJson(element, Project.class); - p.setSCM(SCM.GIT); - p.setScmURL(String.format("git://github.com/%s/%s.git", p.getOwner(), p.getName())); - - projects.add(p); - totalRepositories++; + JsonObject rawJsonObject = jsonArrayLegacy.get(0).getAsJsonObject(); + String stringElement = rawJsonObject.toString(); + Project p = gson.fromJson(stringElement, Project.class); + + p.setSCM(SCM.GIT); + String owner = rawJsonObject.getAsJsonObject().get("owner").getAsString(); + p.setScmURL(String.format("git://github.com/%s/%s.git", owner, p.getName())); + + projects.add(p); + totalRepositories++; + } } JsonElement lastPagesRepository = jsonArray.get(jsonArray.size() -1); since = lastPagesRepository.getAsJsonObject().get("id").getAsInt();