Fix issue #108

PixivModel.py

@@ -23,6 +23,7 @@ class PixivArtist:
     imageList = []
     isLastPage = None
     haveImages = None
+    totalImages = 0
 
     def __init__(self, mid=0, page=None, fromImage=False):
         if page is not None:
@@ -110,6 +111,7 @@ def ParseImages(self, page):
                 # fuck performance :D
                 if href not in self.imageList:
                     self.imageList.append(href)
+        self.totalImages = SharedParser.parseCountBadge(page)
 
     def IsNotLoggedIn(self, page):
         check = page.findAll('a', attrs={'class': 'signup_button'})
@@ -800,10 +802,10 @@ def exportList(l, filename):
 
 class PixivTags:
     '''Class for parsing tags search page'''
-    # imageList = None
     itemList = None
     haveImage = None
     isLastPage = None
+    availableImages = 0
     __re_illust = re.compile(r'member_illust.*illust_id=(\d*)')
     __re_imageItemClass = re.compile(r".*\bimage-item\b.*")
 
@@ -849,6 +851,7 @@ def parseTags(self, page):
                                 imageResponse = temp.contents[1]
                 self.itemList.append(PixivTagsItem(int(image_id), int(bookmarkCount), int(imageResponse)))
         self.checkLastPage(page)
+        self.availableImages = SharedParser.parseCountBadge(page)
         return self.itemList
 
     def parseMemberTags(self, page):
@@ -863,6 +866,7 @@ def parseMemberTags(self, page):
                     image_id = int(result[0])
                     self.itemList.append(PixivTagsItem(int(image_id), 0, 0))
         self.checkLastPage(page, fromMember=True)
+        self.availableImages = SharedParser.parseCountBadge(page)
         return self.itemList
 
     def checkLastPage(self, page, fromMember=False):
@@ -971,3 +975,15 @@ def shortenPixivUrlInBody(self, string):
         string = self.short_pattern.sub("", string).strip()
         string = string + " " + shortened
         return string
+
+class SharedParser:
+    @staticmethod
+    def parseCountBadge(page):
+    # parse image count from count-badge
+        totalImages = 0
+        count_badge_span = page.find('span', attrs={'class':'count-badge'})
+        if count_badge_span is not None:
+            tempCount = re.findall('\d+', count_badge_span.string)
+            if tempCount > 0:
+                totalImages = int(tempCount[0])
+        return totalImages

PixivUtil2.py

@@ -388,7 +388,10 @@ def process_member(mode, member_id, user_dir='', page=1, end_page=0, bookmark=Fa
                 retry_count = 0
                 while True:
                     try:
-                        total_image_page_count = ((page - 1) * 20) + len(artist.imageList)
+                        if artist.totalImages > 0:
+                            total_image_page_count = artist.totalImages
+                        else:
+                            total_image_page_count = ((page - 1) * 20) + len(artist.imageList)
                         title_prefix = "MemberId: {0} Page: {1} Image {2}+{3} of {4}".format(member_id,
                                                                                              page,
                                                                                              no_of_images,
@@ -765,7 +768,10 @@ def process_tags(mode, tags, page=1, end_page=0, wild_card=True, title_caption=F
                     result = 0
                     while True:
                         try:
-                            total_image = ((i - 1) * 20) + len(t.itemList)
+                            if t.availableImages > 0:
+                                total_image = t.availableImages
+                            else:
+                                total_image = ((i - 1) * 20) + len(t.itemList)
                             title_prefix = "Tags:{0} Page:{1} Image {2}+{3} of {4}".format(tags, i, images, skipped_count, total_image)
                             if not member_id is None:
                                 title_prefix = "MemberId: {0} Tags:{1} Page:{2} Image {3}+{4} of {5}".format(member_id,

changelog.txt

@@ -2,7 +2,8 @@
 - Fix unicode decode error when downloading image.
 - Implement #107: add option to specify different database.
 - Update readme.txt
-- Fix Issue #109: update parser.
+- Fix Issue #109: update parser for title and caption.
+- Fix Issue #108: parse total image count.
 
 20160211:
 - Fix Issue #98: Fix error message when download fail.

test.PixivModel.py

@@ -34,6 +34,7 @@ def testPixivArtistProfileDataSrc(self):
       self.assertNotEqual(artist, None)
       self.assertEqual(artist.artistId, 1107124)
       self.assertEqual(artist.artistToken, 'kirabara29')
+      self.assertEqual(artist.totalImages, 69)
 
     def testPixivArtistNoImage(self):
       #print '\nTesting member page - no image'
@@ -428,7 +429,7 @@ def testPixivImageBookmarkMember(self):
       self.assertTrue(42934821 in result)
       self.assertTrue(30431160 in result)
       self.assertTrue(44328684 in result)
-	  
+
 class TestMyPickPage(unittest.TestCase):
     def testMyPickPage(self):
         try:
@@ -490,6 +491,7 @@ def testTagsSearchExact1(self):
 
         self.assertEqual(len(image.itemList), 20)
         self.assertEqual(image.isLastPage, False)
+        self.assertEqual(image.availableImages, 2288)
 
     ## tags.php?tag=%E3%81%93%E3%81%AE%E4%B8%AD%E3%81%AB1%E4%BA%BA%E3%80%81%E5%A6%B9%E3%81%8C%E3%81%84%E3%82%8B%21
     def testTagsSearchExact(self):
@@ -560,10 +562,11 @@ def testTagsMemberSearch(self):
         image = PixivTags()
         image.parseMemberTags(page)
 
-        self.assertEqual(len(image.itemList), 20)
-        self.assertEqual(image.itemList[0].imageId, 25757869)
-        self.assertEqual(image.itemList[19].imageId, 14818847)
+        self.assertEqual(len(image.itemList), 40)
+        self.assertEqual(image.itemList[0].imageId, 53977340)
+        self.assertEqual(image.itemList[19].imageId, 45511597)
         self.assertEqual(image.isLastPage, False)
+        self.assertEqual(image.availableImages, 59)
 
     def testTagsMemberSearchLast(self):
         br = Browser()

test.updateHtml.py

@@ -93,6 +93,8 @@ def main():
         downloadPage('http://www.pixiv.net/search.php?s_mode=s_tag&word=R-18%20K-On!','./test/test-tags-search-skip-showcase.htm')
 
         downloadPage('http://www.pixiv.net/search.php?word=%E3%82%AF%E3%83%89%E3%83%AA%E3%83%A3%E3%83%95%E3%82%AB&s_mode=s_tag_full','./test/test-tags-search-exact2.htm')
+
+        downloadPage('http://www.pixiv.net/member_illust.php?id=313631&tag=R-18', './test/test-tags-member-search.htm')
         ## Not updated:
         ## ./test/test-login-error.htm
         ## ./test/test-member-suspended.htm