[feat](pixiv-engine): v1.2.0

1.修改pixiv_data数据结构 2.修改Dao中insertPixivData使其适于新的数据结构 3.优化getPixivDataTable算法 4.调整getMultiPixivDataByTag算法 5.使getMultiPixivDataByTag方法返回的数据类型为ArrayList 6.修改gradle版本号
mikoto-tech · Jan 24, 2022 · b05cd4c · b05cd4c
1 parent 3db40d0
commit b05cd4c
Show file tree

Hide file tree

Showing 16 changed files with 68 additions and 59 deletions.
diff --git a/.gradle/7.2/executionHistory/executionHistory.bin b/.gradle/7.2/executionHistory/executionHistory.bin
diff --git a/.gradle/7.2/executionHistory/executionHistory.lock b/.gradle/7.2/executionHistory/executionHistory.lock
diff --git a/.gradle/7.2/fileHashes/fileHashes.bin b/.gradle/7.2/fileHashes/fileHashes.bin
diff --git a/.gradle/7.2/fileHashes/fileHashes.lock b/.gradle/7.2/fileHashes/fileHashes.lock
diff --git a/.gradle/7.2/fileHashes/resourceHashesCache.bin b/.gradle/7.2/fileHashes/resourceHashesCache.bin
diff --git a/.gradle/buildOutputCleanup/buildOutputCleanup.lock b/.gradle/buildOutputCleanup/buildOutputCleanup.lock
diff --git a/.gradle/buildOutputCleanup/outputFiles.bin b/.gradle/buildOutputCleanup/outputFiles.bin
diff --git a/.gradle/checksums/checksums.lock b/.gradle/checksums/checksums.lock
diff --git a/.gradle/checksums/md5-checksums.bin b/.gradle/checksums/md5-checksums.bin
diff --git a/.gradle/checksums/sha1-checksums.bin b/.gradle/checksums/sha1-checksums.bin
diff --git a/.gradle/checksums/sha256-checksums.bin b/.gradle/checksums/sha256-checksums.bin
diff --git a/.gradle/checksums/sha512-checksums.bin b/.gradle/checksums/sha512-checksums.bin
diff --git a/build.gradle b/build.gradle
@@ -5,7 +5,7 @@ plugins {
 }
 
 group 'net.mikoto.pixiv'
-version '1.1.1'
+version '1.2.0'
 
 repositories {
     mavenCentral()

diff --git a/src/main/java/net/mikoto/pixiv/engine/dao/PixivDataDao.java b/src/main/java/net/mikoto/pixiv/engine/dao/PixivDataDao.java
@@ -22,17 +22,7 @@ public class PixivDataDao extends BaseDao {
     /**
      * Constants.
      */
-    private static final Integer GRAND_0 = 100000;
-    private static final Integer GRAND_1 = 50000;
-    private static final Integer GRAND_2 = 40000;
-    private static final Integer GRAND_3 = 30000;
-    private static final Integer GRAND_4 = 25000;
-    private static final Integer GRAND_5 = 20000;
-    private static final Integer GRAND_6 = 15000;
-    private static final Integer GRAND_7 = 10000;
-    private static final Integer GRAND_8 = 5000;
-    private static final Integer GRAND_9 = 1000;
-    private static final Integer GRAND_10 = 0;
+    private static final String RESULT = "result";
 
     /**
      * Init method.
@@ -92,20 +82,19 @@ public PixivDataResult queryPixivData(String sql) throws SQLException {
     public void insertPixivData(@NotNull PixivData pixivData) throws SQLException {
         Connection connection = getConnection();
 
-        String sql = "select count(*) as result from " + getPixivDataTable(pixivData.getBookmarkCount()) + " where pk_artwork_id=?";
+        String sql = "select count(*) as result from " + getPixivDataTable(pixivData.getArtworkId()) + " where pk_artwork_id=?";
         PreparedStatement preparedStatement = connection.prepareStatement(sql);
         preparedStatement.setInt(1, pixivData.getArtworkId());
         ResultSet resultSet = preparedStatement.executeQuery();
         resultSet.next();
-        int result = resultSet.getInt("result");
-        if (result != 0) {
+        if (resultSet.getInt(RESULT) != 0) {
             sql = "DELETE FROM " + getPixivDataTable(pixivData.getBookmarkCount()) + " WHERE pk_artwork_id=?;";
             preparedStatement = connection.prepareStatement(sql);
             preparedStatement.setInt(1, pixivData.getArtworkId());
             preparedStatement.executeUpdate();
         }
 
-        sql = "INSERT INTO pixiv_data." + getPixivDataTable(pixivData.getBookmarkCount()) + " (pk_artwork_id, artwork_title, author_id, author_name, description, tags, illust_url_mini, illust_url_thumb, illust_url_small, illust_url_regular, illust_url_original, page_count, bookmark_count, like_count, view_count, grading, update_date, create_date, crawl_date)  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
+        sql = "INSERT INTO pixiv_data." + getPixivDataTable(pixivData.getArtworkId()) + " (pk_artwork_id, artwork_title, author_id, author_name, description, tags, illust_url_mini, illust_url_thumb, illust_url_small, illust_url_regular, illust_url_original, page_count, bookmark_count, like_count, view_count, grading, update_date, create_date, crawl_date)  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);";
         preparedStatement = connection.prepareStatement(sql);
         preparedStatement.setInt(1, pixivData.getArtworkId());
         preparedStatement.setString(2, pixivData.getArtworkTitle());
@@ -141,34 +130,10 @@ public void insertPixivData(@NotNull PixivData pixivData) throws SQLException {
     /**
      * Get table.
      *
-     * @param bookmarkCount Bookmark count.
+     * @param artworkId The id of this artwork.
      * @return A table name.
      */
-    private String getPixivDataTable(@NotNull Integer bookmarkCount) {
-        String table = "bookmark_";
-        if (bookmarkCount > GRAND_0) {
-            table += String.valueOf(GRAND_0);
-        } else if (bookmarkCount > GRAND_1) {
-            table += GRAND_1 + "_" + GRAND_0;
-        } else if (bookmarkCount > GRAND_2) {
-            table += GRAND_2 + "_" + GRAND_1;
-        } else if (bookmarkCount > GRAND_3) {
-            table += GRAND_3 + "_" + GRAND_2;
-        } else if (bookmarkCount > GRAND_4) {
-            table += GRAND_4 + "_" + GRAND_3;
-        } else if (bookmarkCount > GRAND_5) {
-            table += GRAND_5 + "_" + GRAND_4;
-        } else if (bookmarkCount > GRAND_6) {
-            table += GRAND_6 + "_" + GRAND_5;
-        } else if (bookmarkCount > GRAND_7) {
-            table += GRAND_7 + "_" + GRAND_6;
-        } else if (bookmarkCount > GRAND_8) {
-            table += GRAND_8 + "_" + GRAND_7;
-        } else if (bookmarkCount > GRAND_9) {
-            table += GRAND_9 + "_" + GRAND_8;
-        } else if (bookmarkCount >= GRAND_10) {
-            table += GRAND_10 + "_" + GRAND_9;
-        }
-        return table;
+    public String getPixivDataTable(@NotNull Integer artworkId) {
+        return "artwork_table_" + (int) Math.ceil(Double.valueOf(artworkId) / 5000000.0);
     }
 }
diff --git a/src/main/java/net/mikoto/pixiv/engine/service/PixivDataService.java b/src/main/java/net/mikoto/pixiv/engine/service/PixivDataService.java
@@ -13,7 +13,10 @@
 import java.io.IOException;
 import java.sql.SQLException;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Random;
 
 import static net.mikoto.pixiv.engine.util.HttpUtil.httpGet;
 
@@ -157,26 +160,24 @@ public PixivData getPixivDataByTag(@NotNull String tag, @NotNull PixivDataDao pi
      * @return A pixiv data.
      * @throws SQLException An error.
      */
-    public Set<PixivData> getMultiPixivDataByTag(@NotNull String tag, @NotNull PixivDataDao pixivDataDao, Column column, Type type, int count) throws SQLException {
-        ArrayList<String> pageArray = getPageArray(type);
+    public ArrayList<PixivData> getMultiPixivDataByTag(@NotNull String tag, @NotNull PixivDataDao pixivDataDao, Column column, Type type, int count) throws SQLException {
         ArrayList<PixivData> outputPixivDataArrayList = new ArrayList<>();
-        int outputPixivDataCount = 0;
 
         for (String page :
-                pageArray) {
-            String sql = "SELECT * FROM `" + page + "` WHERE `tags` LIKE '%" + tag + "%' order by " + column + " " + type + " limit " + (count - outputPixivDataCount);
+                getPageArray(type)) {
+            String sql = "SELECT * FROM `" + page + "` WHERE `tags` LIKE '%" + tag + "%' order by " + column + " " + type + " limit " + (count - outputPixivDataArrayList.size());
             PixivDataResult pixivDataResult = pixivDataDao.queryPixivData(sql);
-            outputPixivDataCount += pixivDataResult.getPixivDataCount();
             outputPixivDataArrayList.addAll(pixivDataResult.getPixivDataSet());
 
-            if (outputPixivDataCount >= count) {
-                for (int i = 0; i < outputPixivDataCount - count; i++) {
-                    outputPixivDataArrayList.remove(outputPixivDataArrayList.size() - 1);
-                }
-                return new HashSet<>(outputPixivDataArrayList);
+            if (outputPixivDataArrayList.size() >= count) {
+                return outputPixivDataArrayList;
             }
         }
-        return null;
+        if (outputPixivDataArrayList.size() > 0) {
+            return outputPixivDataArrayList;
+        } else {
+            return null;
+        }
     }
 
     /**

diff --git a/src/test/java/net/mikoto/pixiv/engine/PixivEngineTest.java b/src/test/java/net/mikoto/pixiv/engine/PixivEngineTest.java
@@ -1,15 +1,18 @@
 package net.mikoto.pixiv.engine;
 
+import com.alibaba.fastjson.JSON;
 import net.mikoto.pixiv.api.pojo.PixivData;
 import net.mikoto.pixiv.engine.logger.impl.ConsoleTimeFormatLogger;
 import net.mikoto.pixiv.engine.pojo.Config;
 import net.mikoto.pixiv.engine.service.Column;
 import net.mikoto.pixiv.engine.service.Type;
 import org.junit.jupiter.api.Test;
 
+import java.io.IOException;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.Set;
+
+import static net.mikoto.pixiv.engine.util.HttpUtil.httpGet;
 
 /**
  * @author mikoto
@@ -20,7 +23,7 @@ public class PixivEngineTest {
     void test() {
         Config config = new Config();
         config.setLogger(new ConsoleTimeFormatLogger());
-        config.setJpbcUrl("jdbc:mysql://192.168.10.2:3306/pixiv_data");
+        config.setJpbcUrl("jdbc:mysql://192.168.10.110:3306/pixiv_data");
         config.setUserName("pixiv_data");
         config.setUserPassword("YOUR DATABASE PASSWORD");
         config.setKey("1fc499f4ef758ad328505f6747d39198c9373cb1dfe893f21300f0eeb7a3f4c4");
@@ -32,7 +35,7 @@ void test() {
         PixivEngine pixivEngine = new PixivEngine(config);
 
         try {
-            Set<PixivData> pixivDataSet = pixivEngine.getPixivDataService().getMultiPixivDataByTag("³õÒô", pixivEngine.getPixivDataDao(), Column.bookmark_count, Type.desc, 10);
+            ArrayList<PixivData> pixivDataSet = pixivEngine.getPixivDataService().getMultiPixivDataByTag("³õÒô", pixivEngine.getPixivDataDao(), Column.bookmark_count, Type.desc, 5);
             for (PixivData pixivData :
                     pixivDataSet) {
                 config.getLogger().log(pixivData.toJsonObject().toJSONString());
@@ -41,4 +44,44 @@ void test() {
             e.printStackTrace();
         }
     }
+
+    @Test
+    void daoTest() {
+        Config config = new Config();
+        config.setLogger(new ConsoleTimeFormatLogger());
+        config.setJpbcUrl("jdbc:mysql://192.168.10.110:3306/pixiv_data");
+        config.setUserName("pixiv_data");
+        config.setUserPassword("YOUR DATABASE PASSWORD");
+        config.setKey("1fc499f4ef758ad328505f6747d39198c9373cb1dfe893f21300f0eeb7a3f4c4");
+        ArrayList<String> pixivDataForwardServer = new ArrayList<>();
+        pixivDataForwardServer.add("pixiv-forward-1.mikoto-tech.cc");
+        pixivDataForwardServer.add("pixiv-forward-3.mikoto-tech.cc");
+        config.setPixivDataForwardServer(pixivDataForwardServer);
+
+        PixivEngine pixivEngine = new PixivEngine(config);
+
+        config.getLogger().log(pixivEngine.getPixivDataDao().getPixivDataTable(94236849));
+    }
+
+    @Test
+    void datTest2() {
+        Config config = new Config();
+        config.setLogger(new ConsoleTimeFormatLogger());
+        config.setJpbcUrl("jdbc:mysql://192.168.10.110:3306/pixiv_data");
+        config.setUserName("pixiv_data");
+        config.setUserPassword("YOUR DATABASE PASSWORD");
+        config.setKey("1fc499f4ef758ad328505f6747d39198c9373cb1dfe893f21300f0eeb7a3f4c4");
+        ArrayList<String> pixivDataForwardServer = new ArrayList<>();
+        pixivDataForwardServer.add("pixiv-forward-1.mikoto-tech.cc");
+        pixivDataForwardServer.add("pixiv-forward-3.mikoto-tech.cc");
+        config.setPixivDataForwardServer(pixivDataForwardServer);
+
+        PixivEngine pixivEngine = new PixivEngine(config);
+
+        try {
+            pixivEngine.getPixivDataDao().insertPixivData(new PixivData().loadJson(JSON.parseObject(httpGet("https://pixiv-forward-1.mikoto-tech.cc/getArtworkInformation?key=1fc499f4ef758ad328505f6747d39198c9373cb1dfe893f21300f0eeb7a3f4c4&artworkId=94236849"))));
+        } catch (SQLException | IOException e) {
+            e.printStackTrace();
+        }
+    }
 }