-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
094f657
commit 429d27e
Showing
9 changed files
with
765 additions
and
28 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <groupId>com.Agentspades</groupId> | ||
| <artifactId>etsyScraperMaven</artifactId> | ||
| <name>etsyScraperMaven</name> | ||
| <version>1.0</version> | ||
| <build> | ||
| <plugins> | ||
| <plugin> | ||
| <artifactId>maven-shade-plugin</artifactId> | ||
| <version>3.2.4</version> | ||
| <executions> | ||
| <execution> | ||
| <phase>package</phase> | ||
| <goals> | ||
| <goal>shade</goal> | ||
| </goals> | ||
| <configuration> | ||
| <transformers> | ||
| <transformer> | ||
| <mainClass>com.agentspades.etsyscrapermaven.SearchGUI</mainClass> | ||
| </transformer> | ||
| </transformers> | ||
| </configuration> | ||
| </execution> | ||
| </executions> | ||
| </plugin> | ||
| </plugins> | ||
| </build> | ||
| <properties> | ||
| <maven.compiler.target>15</maven.compiler.target> | ||
| <maven.compiler.source>15</maven.compiler.source> | ||
| <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> | ||
| </properties> | ||
| </project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <project-shared-configuration> | ||
| <!-- | ||
| This file contains additional configuration written by modules in the NetBeans IDE. | ||
| The configuration is intended to be shared among all the users of project and | ||
| therefore it is assumed to be part of version control checkout. | ||
| Without this configuration present, some functionality in the IDE may be limited or fail altogether. | ||
| --> | ||
| <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1"> | ||
| <!-- | ||
| Properties that influence various parts of the IDE, especially code formatting and the like. | ||
| You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up. | ||
| That way multiple projects can share the same settings (useful for formatting rules for example). | ||
| Any value defined here will override the pom.xml file value but is only applicable to the current project. | ||
| --> | ||
| <netbeans.hint.licensePath>${project.basedir}/licenseheaderC:\Users\Agentspades\Google Drive\Java\ETSY_Scraper\LICENSE.txt</netbeans.hint.licensePath> | ||
| </properties> | ||
| </project-shared-configuration> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| java -jar etsyScraper-0.1-shaded.jar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,7 +4,7 @@ | |
| * Purpose: Scrape etsy search results to show listing tags | ||
| * Author: Agentspades | ||
| * Date: 1 Jul 2021 | ||
| * ToDo: | ||
| * ToDo: Finish About JOptionPane | ||
| */ | ||
| package com.agentspades.etsyscrapermaven; | ||
|
|
||
|
|
@@ -25,7 +25,7 @@ | |
| import javax.swing.table.TableColumnModel; | ||
| import javax.swing.table.TableModel; | ||
| import javax.swing.table.TableRowSorter; | ||
| import com.agentspades.etsyscrapermaven.SearchGUI; | ||
| import javax.swing.JOptionPane; | ||
|
|
||
| public class SearchGUI extends javax.swing.JFrame { | ||
|
|
||
|
|
@@ -128,6 +128,8 @@ private void initComponents() { | |
| jMenuBar1 = new javax.swing.JMenuBar(); | ||
| jMenu1 = new javax.swing.JMenu(); | ||
| jMenuItem1 = new javax.swing.JMenuItem(); | ||
| jMenu2 = new javax.swing.JMenu(); | ||
| jMenuItem2 = new javax.swing.JMenuItem(); | ||
|
|
||
| setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE); | ||
| setTitle("ETSY | Scraper"); | ||
|
|
@@ -175,8 +177,6 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { | |
| "Title 1", "Title 2" | ||
| } | ||
| )); | ||
| results_JTable.setShowHorizontalLines(true); | ||
| results_JTable.setShowVerticalLines(true); | ||
| jScrollPane2.setViewportView(results_JTable); | ||
|
|
||
| top_JTable.setModel(new javax.swing.table.DefaultTableModel( | ||
|
|
@@ -196,8 +196,6 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { | |
| "Title 1", "Title 2" | ||
| } | ||
| )); | ||
| top_JTable.setShowHorizontalLines(true); | ||
| top_JTable.setShowVerticalLines(true); | ||
| jScrollPane3.setViewportView(top_JTable); | ||
|
|
||
| top_JLabel.setFont(new java.awt.Font("sansserif", 0, 18)); // NOI18N | ||
|
|
@@ -215,6 +213,18 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { | |
|
|
||
| jMenuBar1.add(jMenu1); | ||
|
|
||
| jMenu2.setText("Help"); | ||
|
|
||
| jMenuItem2.setText("About"); | ||
| jMenuItem2.addActionListener(new java.awt.event.ActionListener() { | ||
| public void actionPerformed(java.awt.event.ActionEvent evt) { | ||
| jMenuItem2ActionPerformed(evt); | ||
| } | ||
| }); | ||
| jMenu2.add(jMenuItem2); | ||
|
|
||
| jMenuBar1.add(jMenu2); | ||
|
|
||
| setJMenuBar(jMenuBar1); | ||
|
|
||
| javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane()); | ||
|
|
@@ -295,7 +305,7 @@ private void search_JButtonActionPerformed(java.awt.event.ActionEvent evt) {//GE | |
| progress_JTextArea.setText(""); | ||
| }//end if | ||
| //display the search URL in the JTextArea | ||
| progress_JTextArea.append("Proccessing: " +searchUrl); | ||
| progress_JTextArea.setText("Proccessing: " +searchUrl); | ||
| //start a new thread | ||
| new Thread(new scrapeHtmlUnit()).start(); | ||
| }//end outer if | ||
|
|
@@ -319,7 +329,7 @@ private void search_JButtonActionPerformed(java.awt.event.ActionEvent evt) {//GE | |
| /**********************************************************/ | ||
| private void reset_JButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_reset_JButtonActionPerformed | ||
| //reset the searchUrl | ||
| searchUrl = "https://www.etsy.com/au/search?q="; | ||
| searchUrl = "https://www.etsy.com/search?q="; | ||
| //set the fields to blank | ||
| progress_JTextArea.setText(""); | ||
| search_JTextField.setText(""); | ||
|
|
@@ -336,6 +346,19 @@ private void jMenuItem1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FI | |
| new proxySettingsGUI().setVisible(true); | ||
| }//GEN-LAST:event_jMenuItem1ActionPerformed | ||
|
|
||
| private void jMenuItem2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jMenuItem2ActionPerformed | ||
| //store the about message in a String variable | ||
| String aboutText = "This program has been created by Agentspades " | ||
| + "(July, 2021) under the GNU/GPL V3.0 license." | ||
| + "\nusing the below third party library" | ||
| + "\n\nHtmlUnit v2.5.0 \nhttps://htmlunit.sourceforge.io" | ||
| + "\n\nContact: [email protected]" | ||
| + "\nhttps://github.com/Agentspades/ETSY_Scraper"; | ||
| //show a JOptionPane with about info | ||
| JOptionPane.showMessageDialog(null, aboutText, "About", | ||
| JOptionPane.INFORMATION_MESSAGE); | ||
| }//GEN-LAST:event_jMenuItem2ActionPerformed | ||
|
|
||
| /**********************************************************/ | ||
| // Method: public void clearTable() | ||
| // Purpose: Clears the tables contents | ||
|
|
@@ -371,8 +394,6 @@ public void clearTable(int type){ | |
| /**********************************************************/ | ||
| private class scrapeHtmlUnit implements Runnable{ | ||
| public void run(){ | ||
| //clear the progress window | ||
| progress_JTextArea.setText(""); | ||
| //decalre variables | ||
| HtmlPage resultsPage = null; | ||
| List<String> linkList = new LinkedList<String>(); | ||
|
|
@@ -384,7 +405,7 @@ public void run(){ | |
| //this is to fetch more results to overcome the infinite scrolling | ||
| client.getOptions().setScreenHeight(600000); | ||
| //if if a proxy has been specified if not then use the users IP | ||
| if(!proxyIP.isEmpty() && proxyPort != 0){ | ||
| if(proxyIP == null && proxyPort != 0){ | ||
| ProxyConfig proxyConfig = new ProxyConfig(proxyIP, proxyPort, null); | ||
| client.getOptions().setProxyConfig(proxyConfig); | ||
| }//end if | ||
|
|
@@ -414,9 +435,8 @@ public void run(){ | |
| //set progress text | ||
| progress_JTextArea.setText(progress_JTextArea.getText()+ | ||
| "\nScraping ..."); | ||
| //clear the tables | ||
| //clear the table | ||
| clearTable(1); | ||
| clearTable(2); | ||
| //loop through each link fetched from the search results | ||
| for(String link : linkList){ | ||
| //set String variables to blank | ||
|
|
@@ -469,6 +489,8 @@ public void run(){ | |
| //set progress text | ||
| progress_JTextArea.setText(progress_JTextArea.getText()+ | ||
| "\nSequence Complete!"); | ||
| //reset the searchUrl | ||
| searchUrl = "https://www.etsy.com/search?q="; | ||
| //end the thread | ||
| return; | ||
| }//end run() | ||
|
|
@@ -481,6 +503,8 @@ public void run(){ | |
| // Outputs: void | ||
| /**********************************************************/ | ||
| private void countTags(){ | ||
| //clear the top tags table | ||
| clearTable(2); | ||
| //set progress text | ||
| progress_JTextArea.setText(progress_JTextArea.getText()+ "\nCounting the tags used"); | ||
| //declare empty String variables | ||
|
|
@@ -605,8 +629,10 @@ public void run() { | |
|
|
||
| // Variables declaration - do not modify//GEN-BEGIN:variables | ||
| private javax.swing.JMenu jMenu1; | ||
| private javax.swing.JMenu jMenu2; | ||
| private javax.swing.JMenuBar jMenuBar1; | ||
| private javax.swing.JMenuItem jMenuItem1; | ||
| private javax.swing.JMenuItem jMenuItem2; | ||
| private javax.swing.JScrollPane jScrollPane1; | ||
| private javax.swing.JScrollPane jScrollPane2; | ||
| private javax.swing.JScrollPane jScrollPane3; | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters