license parser currently on beta. rl #56

spgroup · Aug 6, 2013 · ba88aeb · ba88aeb
1 parent 22d4bb5
commit ba88aeb
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 17 deletions.
diff --git a/src/java/main/br/ufpe/cin/groundhog/License.java b/src/java/main/br/ufpe/cin/groundhog/License.java
@@ -25,4 +25,8 @@ public String getName() {
 	public String getEntireContent() {
 		return entireContent;
 	}
+
+	public String toString() {
+		return name;
+	}
 }
diff --git a/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java b/src/java/main/br/ufpe/cin/groundhog/codehistory/GitCodeHistory.java
@@ -48,6 +48,7 @@ public File checkoutToDate(String project, File repositoryFolder, final Date dat
 			FileUtil.getInstance().copyDirectory(repositoryFolder,
 					projectFolder);
 
+			logger.info("Checking out to the specific date..");
 			this.gitClient.checkout(projectFolder, date);
 			return projectFolder;
 		} catch (EmptyProjectAtDateException e) {

diff --git a/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java b/src/java/main/br/ufpe/cin/groundhog/crawler/CrawlGitHub.java
@@ -42,7 +42,7 @@ public File downloadProject(Project project) {
 			this.gitClient.clone(cloneUrl, projectFolder);
 			return projectFolder;
 		} catch (Exception e) {
-			String error = String.format("Unable to download %s (%s) project", project.getName(), project.getURL());
+			String error = String.format("Unable to download %s (%s) project", project.getName(), project.getScmURL());
 			logger.error(error);
 			throw new DownloadExecption(error);
 		}

diff --git a/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java b/src/java/main/br/ufpe/cin/groundhog/crawler/ForgeCrawler.java
@@ -9,7 +9,6 @@
 import java.util.concurrent.Future;
 
 import br.ufpe.cin.groundhog.Project;
-import br.ufpe.cin.groundhog.main.JsonInput;
 
 /**
  * An abstract class that defines the forge crawl functionality.
@@ -26,7 +25,7 @@ public abstract class ForgeCrawler {
 	 * Constructs a new ForgeCrawler with a given destinationFolder.
 	 */
 	protected ForgeCrawler() {
-		this.ex = Executors.newFixedThreadPool(JsonInput.getMaxThreads());
+		this.ex = Executors.newCachedThreadPool();
 	}
 
 	/**

diff --git a/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java b/src/java/main/br/ufpe/cin/groundhog/parser/license/LicenseParser.java
@@ -65,8 +65,7 @@ private License extractLicense(String content) {
 
 		for (String license : Licenses.names()) {
 			if (content.contains(license)) {
-				logger.info(String.format("License found! %s uses %s license.",
-						root.getName(), license));
+				logger.info(String.format("License found! %s uses %s license.", root.getName(), license));
 				return new License(license);
 			}
 		}
@@ -75,8 +74,7 @@ private License extractLicense(String content) {
 
 	private boolean containsLicenseWord(String content) {
 
-		for (String licenseKeyword : Lists.newArrayList("license", "copyright",
-				"permission")) {
+		for (String licenseKeyword : Lists.newArrayList("license", "copyright", "permission")) {
 			if (content.toLowerCase().contains(licenseKeyword)) {
 				return true;
 			}

diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java
@@ -355,30 +355,34 @@ public List<Project> getAllForgeProjects(int start, int limit) throws SearchExce
 			int totalRepositories = 0;
 			while(totalRepositories < limit || limit < 0){
 
-				String searchUrl = ROOT + String.format("/repositories?since=%s&language=java&%s", since, this.oauthToken);
+				String searchUrl = String.format("%s/repositories?since=%s&language=java&%s", ROOT, since, this.oauthToken);
 				String jsonString = requests.get(searchUrl);
 
 				JsonArray jsonArray = gson.fromJson(jsonString, JsonElement.class).getAsJsonArray();
 				for (int i = 0; i < jsonArray.size() && 
 						(totalRepositories + i < limit || limit < 0); i++) {
 
 					String repoName = jsonArray.get(i).getAsJsonObject().get("name").getAsString();					
-					String searchUrlLegacy = String.format("%s/legacy/repos/search/%s?language=java&%s", ROOT, repoName, this.oauthToken);
+					String searchUrlLegacy = String.format("%s/legacy/repos/search/%s?language=java%s", ROOT, repoName, this.oauthToken);
 
 					String jsonLegacy = requests.get(searchUrlLegacy);
 
 					JsonObject jsonObject = gson.fromJson(jsonLegacy, JsonElement.class).getAsJsonObject();			
 					JsonArray jsonArrayLegacy = jsonObject.get("repositories").getAsJsonArray();
 
-					if( jsonArrayLegacy.size() == 0 ) continue; // not a java project
+					if( jsonArrayLegacy.size() > 0) {
 
-					String element = jsonArrayLegacy.get(0).toString(); // results are sorted by best match
-					Project p = gson.fromJson(element, Project.class);
-					p.setSCM(SCM.GIT);
-					p.setScmURL(String.format("git://github.com/%s/%s.git", p.getOwner(), p.getName()));
-
-					projects.add(p);
-					totalRepositories++;
+						JsonObject rawJsonObject = jsonArrayLegacy.get(0).getAsJsonObject();
+						String stringElement = rawJsonObject.toString();
+						Project p = gson.fromJson(stringElement, Project.class);
+
+						p.setSCM(SCM.GIT);
+						String owner = rawJsonObject.getAsJsonObject().get("owner").getAsString();
+						p.setScmURL(String.format("git://github.com/%s/%s.git", owner, p.getName()));
+
+						projects.add(p);
+						totalRepositories++;
+					}
 				}				
 				JsonElement lastPagesRepository = jsonArray.get(jsonArray.size() -1);
 				since = lastPagesRepository.getAsJsonObject().get("id").getAsInt();