-
Notifications
You must be signed in to change notification settings - Fork 9
/
checkwiki.pl
8119 lines (6941 loc) · 283 KB
/
checkwiki.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/local/bin/perl -w
#################################################################
# Program: checkwiki.pl
# Descrition: Scan all pages of a Wikipedia-Project (dump or live) for errors
# Author: Stefan Kühn
# Licence: GPL
#################################################################
#################################################################
# Syntax
# perl checkwiki.pl -p enwiki -m live
#################################################################
# New features, last changes and discussion
# http://de.wikipedia.org/wiki/Benutzer:Stefan_Kühn/Check_Wikipedia
#################################################################
# Error exception
$SIG{__DIE__} = \&die_error;
$SIG{__WARN__} = \&warn_error;
use strict;
use warnings;
our $VERSION = '2013-02-15';
#################################################################
# notice
# delete_old_errors_in_db --> Problem with deleting of errors in loadmodus
# delete_deleted_article_from_db --> Problem old articles
#################################################################
# Load Modules
#################################################################
use DBI;
use Getopt::Long qw(:config bundling no_auto_abbrev no_ignore_case);
use LWP::UserAgent;
use URI::Escape;
#################################################################
# declare_global_directorys
#################################################################
our $dump_directory = '/mnt/user-store/dumps/store/'; # toolserver
our $dump_directory2 = '/mnt/user-store/dumps/tmp/';
# our $dump_directory = '../../dump/'; # home or usb
our $output_directory = '/mnt/user-store/sk/data/checkwiki/';
our $input_directory_new = '/mnt/user-store/sk/data/new_article/';
our $input_directory_change = '/mnt/user-store/sk/data/last_changes/';
our $output_templatetiger = '/mnt/user-store/sk/data/templatetiger/';
our $output_geo = '/mnt/user-store/sk/data/geo/';
#our $dump_filename = '/mnt/user-store/dump/dewiki-20080607-pages-articles.xml'; #'Wikipedia-20080502083556.xml';
our $dump_filename = '';
#$dump_filename ='../../dump/dewiki-20071217-pages-articles.xml';
#################################################################
# Declaration of variables (global)
#################################################################
our $quit_program = 'no'; # quit the program (yes,no), for quit the programm in an emergency
our $quit_reason = ''; # quit the program reason
our $test_programm = 'true'; # only for program tests
our $dump_or_live = ''; # scan modus (dump, live, only)
our $silent_modus = 0; # silent modus (very low output at screen) for batch
our $test_modus = 0; # silent modus (very low output at screen) for batch
our $starter_modus = 0; # to update in the loadmodus the cw_starter table
our $load_modus_done = 1; # done article from db
our $load_modus_new = 1; # new article from db
our $load_modus_dump = 1; # new article from db
our $load_modus_last_change = 1; # last_change article from db
our $load_modus_old = 1; # old article from db
our $details_for_page = 'no'; # yes/no durring the scan you can get more details for a article scan
our $time_start = time(); # start timer in secound
our $time_end = time(); # end time in secound
our $date = 0; # date of dump "20060324"
our $line_number = 0; # number of line in dump
our $project; # name of the project 'dewiki'
our $language = ''; # language of dump 'de', 'en';
our $page_number = 0; # number of pages in namesroom 0
our $base = ''; # base of article, 'http://de.wikipedia.org/wiki/Hauptseite'
our $home = ''; # base of article, 'http://de.wikipedia.org/wiki/'
our @namespace; # namespace values
# 0 number
# 1 namespace in project language
# 2 namespace in english language
our $namespaces_count = -1; # number of namespaces
our @namespacealiases; # namespacealiases values
# 0 number
# 1 namespacealias
our $namespacealiases_count= -1; # number of namespacealiases
our @namespace_cat; #all namespaces for categorys
our @namespace_image; #all namespaces for images
our @namespace_templates; #all namespaces for templates
our @magicword_defaultsort;
our @magicword_img_thumbnail;
our @magicword_img_manualthumb;
our @magicword_img_right;
our @magicword_img_left;
our @magicword_img_none;
our @magicword_img_center;
our @magicword_img_framed;
our @magicword_img_frameless;
our @magicword_img_page;
our @magicword_img_upright;
our @magicword_img_border;
our @magicword_img_sub;
our @magicword_img_super;
our @magicword_img_link;
our @magicword_img_alt;
our @magicword_img_width;
our @magicword_img_baseline;
our @magicword_img_top;
our @magicword_img_text_top;
our @magicword_img_middle;
our @magicword_img_bottom;
our @magicword_img_text_bottom;
# Wiki-special variables
our @live_article; # to-do-list for live (all articles to scan)
our $current_live_article = -1; # line_number_of_current_live_article
our $number_of_live_tests = -1; # Number of articles for live test
our $current_live_error_scan = -1; # for scan every 100 article of an error
our @live_to_scan ; # article of one error number which should be scanned
our $number_article_live_to_scan = -1; # all article from one error
our @article_was_scanned; #if an article was scanned, this will insert here
our $xml_text_from_api = ''; # the text from more then one articles from the API
our $error_counter = -1; # number of found errors in all article
our @error_description; # Error Description
# 0 priority in script
# 1 title in English
# 2 description in English
# 3 number of found (only live scanned)
# 4 priority of foreign language
# 5 title in foreign language
# 6 description in foreign language
# 7 number of found in last scan (from statistic file)
# 8 all known errors (from statistic file + live)
# 9 XHTML translation title
# 10 XHTML translation description
our $number_of_error_description = -1; # number of error_description
our $max_error_count = 50; # maximum of shown article per error
our $maximum_current_error_scan = -1; # how much shold be scanned for reach the max_error_count
our $rest_of_errors_not_scan_yet = '';
our $number_of_all_errors_in_all_articles = 0; #all errors
our $for_statistic_new_article = 0;
our $for_statistic_last_change_article = 0;
our $for_statistic_geo_article = 0;
our $for_statistic_number_of_articles_with_error = 0;
###########################
# files
###########################
our $live_filename = 'input_for_live.txt';
our $output_live_wiki = 'output_for_wikipedia.txt';
our $output_dump_wiki = 'output_for_wikipedia_dump.txt';
our $error_list_filename = 'error_list.txt';
our $error_list_filename_only = 'error_list_only.txt';
our $error_list_filename_dump = 'error_list_dump.txt'; #all errors from the last dump scan
our $error_list_filename_backup = 'error_list_dump_backup.txt';
our $error_statistic_filename = 'error_statistic.txt';
our $error_statistic_filename_only = 'error_statistic_only.txt';
our $error_statistic_filename_list = 'error_statistic_list.txt';
our $translation_file = 'translation.txt';
our $error_list_filename_30 = 'error_list_error_030.txt';
our $error_list_filename_every = 'error_list_error'; # for all errors
our $error_geo_list_filename = 'error_geo_list.txt';
our $error_geo_list_filename_only = 'error_geo_list_only.txt';
our $error_geo_list_filename_html = 'error_geo_list.htm';
our $error_geo_list_filename_only_html = 'error_geo_list_only.htm';
our $log_file = 'log.txt';
our $templatetiger_filename = '';
our @inter_list = ( 'af', 'als', 'an', 'ar',
'bg', 'bs',
'ca', 'cs', 'cy',
'da', 'de',
'el', 'en', 'eo', 'es', 'et', 'eu',
'fa', 'fi', 'fr', 'fy',
'gl', 'gv',
'he', 'hi', 'hr', 'hu',
'id', 'is', 'it',
'ja', 'jv',
'ka', 'ko',
'la', 'lb', 'lt',
'ms',
'nds', 'nds_nl', 'nl', 'nn', 'no',
'pl', 'pt',
'ro', 'ru',
'sh', 'simple', 'sk', 'sl', 'sr', 'sv', 'sw',
'ta', 'th', 'tr',
'uk', 'ur',
'vi', 'vo',
'yi',
'zh'
);
our @foundation_projects = ( 'wikibooks', 'b',
'wiktionary', 'wikt',
'wikinews', 'n',
'wikiquote', 'q',
'wikisource', 's',
'wikipedia', 'w',
'wikispecies', 'species',
'wikimedia', 'foundation', 'wmf',
'wikiversity', 'v',
'commons',
'meta', 'metawikipedia', 'm',
'incubator',
'mw',
'quality',
'bugzilla', 'mediazilla',
'nost',
'testwiki'
);
# current time
our ($akSekunden, $akMinuten, $akStunden, $akMonatstag, $akMonat,
$akJahr, $akWochentag, $akJahrestag, $akSommerzeit) = localtime(time);
our $CTIME_String = localtime(time);
$akMonat = $akMonat + 1;
$akJahr = $akJahr + 1900;
$akMonat = "0".$akMonat if ($akMonat<10);
$akMonatstag = "0".$akMonatstag if ($akMonatstag<10);
$akStunden = "0".$akStunden if ($akStunden<10);
$akMinuten = "0".$akMinuten if ($akMinuten<10);
our $translation_page = ''; # name of the page with translation for example in de: "Wikipedia:WikiProject Check Wikipedia/Übersetzung"
our $start_text = '';
$start_text = $start_text ."The WikiProject '''Check Wikipedia''' will help to clean up the syntax of Wikipedia and to find some other errors.\n";
$start_text = $start_text ."\n";
$start_text = $start_text ."'''Betatest''' - At the moment the script has some bugs and not every error on this page is an actual error. \n";
$start_text = $start_text ."\n";
our $description_text = '';
$description_text = $description_text ."== Project description in English == \n";
$description_text = $description_text ."* '''What is the goal of this project?'''\n";
$description_text = $description_text ."** This project should help to clean up the data of all articles in many different languages.\n";
$description_text = $description_text ."** If we have a clear and clean syntax in all articles more projects (for example: Wikipedia-DVD) can use our data more easily.\n";
$description_text = $description_text ."** The project was inspired by [[:en:Wikipedia:WikiProject Wiki Syntax]].\n";
$description_text = $description_text ."** In order to use the data of a Wikipedia project without the Mediawiki software you need to write a parser. If many articles include wrong syntax it is difficult to program the parser since it needs to be complex enough to recognize the syntax errors.\n";
$description_text = $description_text ."** This project helps to find many errors in all kinds of language and will support many languages in the future. \n";
$description_text = $description_text ."\n";
$description_text = $description_text ."* '''How does it work?'''\n";
$description_text = $description_text ."** The script scans every new [http://dumps.wikimedia.org dump] and creates a list of articles with errors.\n";
$description_text = $description_text ."** The script scans all articles on the list on a daily basis to create a new list for users, omitting already-corrected articles.\n";
$description_text = $description_text ."** The script is written in Perl by: [[:de:User:Stefan Kühn|Stefan Kühn]] "."\n";
$description_text = $description_text ."** You can download the script [http://toolserver.org/~sk/checkwiki/checkwiki.pl here]. It is licensed under GPL."."\n";
$description_text = $description_text ."** [[:de:User:Stefan Kühn/Check Wikipedia|New features, last changes and discussion]]. "."\n";
$description_text = $description_text ."\n";
$description_text = $description_text ."* '''What can you do?'''\n";
$description_text = $description_text ."** The script creates a new error page at the toolserver every day. Please copy and paste the daily updated page at the toolserver (See downloads) to this page here. Attention: That page is a UTF-8 document. In case your browser cannot display the file in UTF-8 you can copy it into a text editor (for example: Notepad++) and convert it to UTF-8. \n";
$description_text = $description_text ."** You can fix an error in one or more articles. \n";
$description_text = $description_text ."** You can delete all fixed articles from this list. \n";
$description_text = $description_text ."** If all articles in one category have been fixed you can delete this category. \n";
$description_text = $description_text ."** You can suggest a new category of errors to the author of the script. \n";
$description_text = $description_text ."** You can also inform the author if you want this project to be implemented into your language's Wikipedia. \n";
$description_text = $description_text ."\n";
$description_text = $description_text ."* '''Please don't… '''\n";
$description_text = $description_text ."** insert an article by hand since it will disappear from the list with the next automatic update of this page. \n";
$description_text = $description_text ."** try to fix spelling mistakes within this page since all manual changes will disappear as well with the next update. Instead, send an e-mail or message to the author so he can fix the spelling in the script. \n";
$description_text = $description_text ."\n";
our $category_text = '';
our $top_priority_script = 'Top priority';
our $top_priority_project = '';
our $middle_priority_script = 'Middle priority';
our $middle_priority_project = '';
our $lowest_priority_script = 'Lowest priority';
our $lowest_priority_project = '';
our $dbh; # DatenbaaseHandler
###############################
# variables for one article
###############################
$page_number = $page_number + 1;
our $title = ''; # title of the current article
our $page_id = -1; # page id of the current article
our $revision_id = -1; # revision id of the current article
our $revision_time = -1; # revision time of the current article
our $text = ''; # text of the current article (for work)
our $text_origin = ''; # text of the current article origin (for save)
our $text_without_comments = ''; # text of the current article without_comments (for save)
our $page_namespace = -100; # namespace of page
our $page_is_redirect = 'no';
our $page_is_disambiguation = 'no';
our $page_categories = '';
our $page_interwikis = '';
our $page_has_error = 'no'; # yes/no error in this page
our $page_error_number = -1; # number of all article for this page
our @comments; # 0 pos_start
# 1 pos_end
# 2 comment
our $comment_counter = -1; #number of comments in this page
our @category; # 0 pos_start
# 1 pos_end
# 2 category Test
# 3 linkname Linkname
# 4 original [[Category:Test|Linkname]]
our $category_counter = -1;
our $category_all = ''; # all categries
our @interwiki; # 0 pos_start
# 1 pos_end
# 2 interwiki Test
# 3 linkname Linkname
# 4 original [[de:Test|Linkname]]
# 5 language
our $interwiki_counter = -1;
our @lines; # text seperated in lines
our @headlines; # headlines
our @section; # text between headlines
undef(@section);
our @lines_first_blank; # all lines where the first character is ' '
our @templates_all; # all templates
our @template; # templates with values
# 0 number of template
# 1 templatename
# 2 template_row
# 3 attribut
# 4 value
our $number_of_template_parts = -1; # number of all template parts
our @links_all; # all links
our @images_all; # all images
our @isbn; # all ibsn of books
our @ref; # all ref
our $page_has_geo_error = 'no'; # yes/no geo error in this page
our $page_geo_error_number = -1; # number of all article for this page
our $end_of_dump = 'no'; # when last article from dump scan then 'yes', else 'no'
our $end_of_live = 'no'; # when last article from live scan then 'yes', else 'no'
our $statistic_online_page = -1; # number of pages online from metadata-statistic
check_input_arguments();
open_db();
open_file() if ($quit_program eq 'no'); # logfile, dumpfile, metadata (API, File)
get_error_description() if ($quit_program eq 'no'); # all errordescription from this script
load_text_translation() if ($quit_program eq 'no'); # load translation from wikipage
output_errors_desc_in_db() if ($quit_program eq 'no'); # update the database with newest error description
output_text_translation_wiki() if ($quit_program eq 'no'); # output the new wikipage for translation
load_article_for_live_scan() if ($quit_program eq 'no'); # only for live
scan_pages() if ($quit_program eq 'no'); # scan all aricle
close_file(); # close dump or templatetiger-file
update_table_cw_error_from_dump() if ($quit_program eq 'no');
delete_deleted_article_from_db() if ($quit_program eq 'no');
delete_article_from_table_cw_new() if ($quit_program eq 'no');
delete_article_from_table_cw_change() if ($quit_program eq 'no');
update_table_cw_starter();
#output_errors() if ($quit_program eq 'no');
output_little_statistic() if ($quit_program eq 'no'); # print counter of found errors
output_duration() if ($quit_program eq 'no'); # print time at the end
print $quit_reason if ($quit_reason ne '');
close_db();
close_logfile();
print 'finish'."\n";
#################################################################
#################################################################
#################################################################
#################################################################
#################################################################
sub get_time_string{
my ($aakSekunden, $aakMinuten, $aakStunden, $aakMonatstag, $aakMonat,
$aakJahr, $aakWochentag, $aakJahrestag, $aakSommerzeit) = localtime(time);
$aakMonat = $aakMonat + 1;
$aakJahr = $aakJahr + 1900;
$aakMonat = "0".$aakMonat if ($aakMonat<10);
$aakMonatstag = "0".$aakMonatstag if ($aakMonatstag<10);
$aakStunden = "0".$aakStunden if ($aakStunden<10);
$aakMinuten = "0".$aakMinuten if ($aakMinuten<10);
$aakSekunden = "0".$aakSekunden if ($aakSekunden<10);
my $result = $aakJahr.$aakMonat.$aakMonatstag.' '.$aakStunden.$aakMinuten.$aakSekunden;
return($result);
}
sub check_input_arguments {
my $load_mode;
if (!GetOptions ('load=s' => \$load_mode,
'm=s' => \$dump_or_live,
'p=s' => \$project,
'silent' => \$silent_modus,
'starter' => \$starter_modus,
'test' => \$test_modus)) {
return;
}
# Check argument value for scan mode.
if ($dump_or_live ne 'dump' &&
$dump_or_live ne 'only' &&
$dump_or_live ne 'live') {
$quit_reason .= "Mode unknown, for example: \"-m dump/live/only\".\n\n";
$quit_program = 'yes';
}
# Check that a project name is given.
if (!defined ($project)) {
$quit_reason .= "No project name, for example: \"-p dewiki\".\n\n";
$quit_program = 'yes';
}
# Split load mode.
if (defined ($load_mode) && $dump_or_live eq 'live') {
my %LoadOptions = map { $_ => 1; } split (/\//, $load_mode);
$load_modus_done = exists ($LoadOptions {'done'}); # done article from db
$load_modus_new = exists ($LoadOptions {'new'}); # new article from db
$load_modus_dump = exists ($LoadOptions {'dump'}); # new article from db
$load_modus_last_change = exists ($LoadOptions {'last_change'}); # last_change article from db
$load_modus_old = exists ($LoadOptions {'old'}); # old article from db
}
if ($quit_program eq 'yes') {
# End of script, because no correct parameter
$quit_reason .= "Use for scan a dump\n";
$quit_reason .= "perl checkwiki.pl -p dewiki -m dump\n";
$quit_reason .= "perl checkwiki.pl -p nds_nlwiki -m dump\n";
$quit_reason .= "perl checkwiki.pl -p nds_nlwiki -m dump --silent\n";
$quit_reason .= "perl checkwiki.pl -p nds_nlwiki -m dump --silent --test\n\n";
$quit_reason .= "Use for scan a list of pages live\n";
$quit_reason .= "perl checkwiki.pl -p dewiki -m live\n";
$quit_reason .= "perl checkwiki.pl -p dewiki -m live --silent\n";
$quit_reason .= "perl checkwiki.pl -p dewiki -m live --silent --test\n";
$quit_reason .= "perl checkwiki.pl -p dewiki -m live --load new/done/dump/last_change/old\n";
$quit_reason .= "\n";
} else {
$language = $project;
$language =~ s/source$//;
$language =~ s/wiki$//;
print "\n";
if (!$silent_modus) {
print "#########################################################\n";
print '######## checkwiki.pl - Version '.$VERSION.' ########'."\n";
print "#########################################################\n";
}
two_column_display('start:', $akJahr.'-'.$akMonat.'-'.$akMonatstag.' '.$akStunden.':'.$akMinuten);
two_column_display('project:', $project);
if (!$silent_modus) {
my $modus_output = '';
$modus_output = 'scan a dump' if ($dump_or_live eq 'dump');
$modus_output = 'scan live' if ($dump_or_live eq 'live');
$modus_output = 'scan a dump only some errors' if ($dump_or_live eq 'only');
two_column_display ('Modus:', $dump_or_live. ' ('.$modus_output.')');
}
if ($test_modus) { #modus only for test
$project .= '_test';
two_column_display ('Test-Modus:', $project.'!!!');
}
}
}
sub open_db{
#################################################################
# DB
#################################################################
#load password from local file
open(PWD, "</home/sk/.mytop");
my $password = '';
do {
my $test = <PWD>;
if ($test =~ /^pass=/ ) {
$password = $test;
$password =~ s/^pass=//g;
$password =~ s/\n//g;
}
}
while (eof(PWD) != 1);
close(PWD);
my $hostname = `hostname`; # get name of host (PC-Name)
chomp($hostname);
two_column_display ('host:', $hostname);
#Connect to database u_sk_yarrow
if ( $hostname =~ 'kunopc'){
$dbh = DBI->connect( 'DBI:mysql:u_sk_yarrow', # local
'sk',
$password ,
{
RaiseError => 1,
AutoCommit => 1
}
) or die "Database connection not made: $DBI::errstr" . DBI->errstr;
} else {
$dbh = DBI->connect( 'DBI:mysql:u_sk_yarrow:host=sql', # Toolserver
'sk',
$password ,
{
RaiseError => 1,
AutoCommit => 1
}
) or die "Database connection not made: $DBI::errstr" . DBI->errstr;
}
$password = '';
}
sub close_db{
# close database
$dbh->disconnect();
}
sub close_logfile{
# close logfile
close (LOGFILE) if (!$starter_modus);
}
###################################################################################
sub get_error_description{
# this subroutine check out the error description of all possible errors
print_line();
two_column_display ('load:', 'all error description from script');
error_list('get_description');
# count the number of error description
$number_of_error_description = 1; # first error is error with number 1
while (defined($error_description[$number_of_error_description][1]) ) {
#print $number_of_error_description.' '. $error_description[$number_of_error_description][1]."\n";
$number_of_error_description = $number_of_error_description + 1;
}
# set all known error description to a basic level
for (my $i = 1; $i <= $number_of_error_description; $i++) {
#$error_description[$i][0] = -1; # set in error
#$error_description[$i][1] = ''; # set in error
#$error_description[$i][2] = ''; # set in error
$error_description[$i][3] = 0;
$error_description[$i][4] = -1;
$error_description[$i][5] = '';
$error_description[$i][6] = '';
$error_description[$i][7] = 0;
$error_description[$i][8] = 0;
$error_description[$i][9] = '';
$error_description[$i][10] = '';
}
my $output_number = $number_of_error_description -1;
two_column_display ('error description:', $output_number.' in script');
}
###################################################################################
sub open_file{
# create subdirectory
#print $output_directory.$project."\n";
if (not (-e $output_directory.$project )) {
print 'create directory:'."\t". $output_directory.$project."\n";
#mkdir($output_directory.$project ,0777);
system ('mkdir -p '.$output_directory.$project);
}
################################
# open logfile
my $log_filename = $output_directory.$project.'/'.$project.'_'.$log_file;
open (LOGFILE, '+>'.$log_filename) if (!$starter_modus);
################################
# if new dump is available
if ($dump_or_live eq 'dump') {
$dump_filename = search_for_last_dump();
two_column_display ('Dump_filename:', $dump_filename) if (!$silent_modus);
my $last_dump_filename = $output_directory.$project.'/'.$project.'_last_dump_name.txt';
two_column_display ('last_dump_filename:', $output_directory.$project.'/');
two_column_display ('', $project.'_last_dump_name.txt');
if (not (-e $last_dump_filename)) {
# create the file if not exist
system ('touch '.$last_dump_filename);
two_column_display ('create last_dump_file:', $project.'_last_dump_name.txt');
open (LAST_DUMP_NAME_FIRST, '+>'.$last_dump_filename);
print LAST_DUMP_NAME_FIRST 'x';
close(LAST_DUMP_NAME_FIRST);
}
#read the last name
#print 'check old dumpname'."\n";
open (LAST_DUMP_NAME, '<'.$last_dump_filename);
my $last_dump_name_old = '';
$last_dump_name_old = <LAST_DUMP_NAME>;
$last_dump_name_old = '' if not defined;
chomp ($last_dump_name_old);
close(LAST_DUMP_NAME);
#get date from dumpfile
our $dump_date_for_output = $dump_filename;
$dump_date_for_output =~ s/^[^\-]-//g;
$dump_date_for_output =~ s/^[^0-9]+//g;
$dump_date_for_output =~ s/[^0-9]+$//g;
if (length($dump_date_for_output) >=8){
$dump_date_for_output = substr($dump_date_for_output,0,4).'-'.substr($dump_date_for_output,4,2).'-'.substr($dump_date_for_output,6,2);
}
#print $dump_date_for_output."\n";
if ($dump_filename ne $last_dump_name_old ) {
# if not the newest dump then start dump scan
two_column_display ('Last scanned dump:', $last_dump_name_old);
two_column_display ('Current found dump:', $dump_filename);
open (LAST_DUMP_NAME, '>'.$last_dump_filename);
print LAST_DUMP_NAME $dump_filename;
close(LAST_DUMP_NAME);
#print 'nice -n 5 perl checkwiki.pl -p '.$project.' -m dump' ."\n";
# if ($dump_or_live eq 'live') {
# print "\n\n";
# system ('nice -n 5 perl checkwiki.pl -p '.$project.' -m dump --silent') ;
# print "\n\n";
# }
}
#update last_dump time for project in database
my $sql_text = "update /* SLOW_OK */ cw_project set last_dump ='".$dump_date_for_output."' where project = '". $project ."';";
my $sth = $dbh->prepare( $sql_text );
$sth->execute;
#delete old list of articles from last dumpscan in table cw_dumpscan
my $sql_text2 = "delete /* SLOW_OK */ from cw_dumpscan where project = '". $project ."';";
$sth = $dbh->prepare( $sql_text2 );
$sth->execute;
}
################################
if ($dump_or_live eq 'dump' or $dump_or_live eq 'only') {
#print "last=x".$dump_filename."x\n";
# check for existens dump
my $full_dump_path_filename = $dump_directory.$project.'/'.$dump_filename;
if (not -e $full_dump_path_filename) {
$full_dump_path_filename = $dump_directory2.'/'.$dump_filename;
}
#print $full_dump_path_filename."\n";
if ($dump_filename ne '' and -e $full_dump_path_filename ) {
#print 'Data: '."\t\t"."$dump_directory$dump_filename\n";
#open dump
open(DUMP, "bzip2 -d -q <$full_dump_path_filename |");
#read_and_write_metadata_from_url();
} else {
$quit_program = 'yes';
$quit_reason = $quit_reason. "file '$full_dump_path_filename'". " don't exist!\n";
}
#################
# Templatetiger
#################
$templatetiger_filename = $output_templatetiger.$project.'/'.$project.'_templatetiger.txt';
if (not (-e $output_templatetiger.$project )) {
two_column_display ('create new subdirectory', 'templatetiger');
system ('mkdir -p '.$output_templatetiger.$project);
}
if (-e $templatetiger_filename ) {
two_column_display ('delete old TT-file:', $project.'_templatetiger.txt');
system ('rm -f '.$templatetiger_filename) ;
}
open (TEMPLATETIGER, '>>'.$templatetiger_filename);
#################
# GEO Export
#################
our $geo_export_filename = $output_geo.$project.'/'.$project.'_coordinates.txt';
if (not (-e $output_geo.$project )) {
two_column_display ('create new subdirectory', 'geo');
#mkdir($output_geo.$project ,0777);
system ('mkdir -p '.$output_geo.$project);
}
if (-e $geo_export_filename ) {
two_column_display ('Delete old Geo-file', $geo_export_filename);
system ('rm -f '.$geo_export_filename) ;
}
}
# delete old error_list
if ($quit_program eq 'no' ) {
read_and_write_metadata_from_url();
load_metadata_from_file();
}
}
sub search_for_last_dump {
# search in dump_directory for the last XML-file of a project
my $last_file ='';
print_line();
two_column_display ('search dump in:', $dump_directory);
two_column_display ('search dump in:', $dump_directory2);
my @xml_files1 = glob($dump_directory .$project.'/*-pages-articles.xml.bz2'); # ../store
my @xml_files2 = glob($dump_directory2.$project.'*-pages-articles.xml.bz2'); # ../tmp
my @xml_files = (@xml_files1, @xml_files2); # add both file-arrays
my $count_xml_files = @xml_files;
for (my $i = 0; $i < $count_xml_files; $i++) {
# List of all xml-files in dump_directory
my $byte = -s $xml_files[$i];
#print $xml_files[$i].' '.$byte."\n";
$xml_files[$i] =~ s/(.)+\///g;
my $project_test = $project;
$project_test =~ s/_test$//;
if (( index($xml_files[$i], $project.'-') == 0 # only this project
or index($xml_files[$i], $project_test.'-') == 0 ) #
and $byte > 0 ) { # only more then 0 bytes files
#the last project dump (more then 0 byte)
if ($xml_files[$i] =~ /^$project(_test)?-[0-9]/) {
#print "\t".$xml_files[$i]."\n";
$last_file = $xml_files[$i];
}
}
}
if ($last_file eq '' and $dump_or_live ne 'live') { # stop if dump scan , run if the program will scan live
# No file found
$quit_program = 'yes';
$quit_reason = $quit_reason.$count_xml_files.' XML-files found in folder '.$dump_directory."\n";
$quit_reason = $quit_reason.'Found no XML-file for project: '.$project."\n";
}
@xml_files = (); # free memory
return($last_file);
}
######################################################################
sub load_article_for_live_scan{
if ($dump_or_live eq 'live' ) {
# open list for live
print_line();
two_column_display('Load article for:', 'live scan') if (!$silent_modus);
#print 'Data: '."\t\t".$output_directory.$project.'/'.$project.'_'.$error_list_filename ."\n";
if (not (-e $output_directory.$project.'/'.$project.'_'.$error_list_filename )){
#$quit_program = 'yes';
#$quit_reason = $quit_reason. "file:" .$output_directory.$project.'/'.$project.'_'.$error_list_filename. " don't exist!\n";
#print 'file:' .$output_directory.$project.'/'.$project.'_'.$error_list_filename. " don't exist!\n";
print 'create '.$output_directory.$project.'/'.$project.'_'.$error_list_filename. "\n";
system ('touch '.$output_directory.$project.'/'.$project.'_'.$error_list_filename);
} else {
#read articles(live)
new_article(250) if ($load_modus_new); # get 250 new article last days
last_change_article(50) if ($load_modus_last_change); # get 10 change article last days
get_done_article_from_database(250) if ($load_modus_done); # get 250 article which are set as done in the database
# which are not scan_live - NEW: with table cw_dumpscan
get_oldest_article_from_database(250) if ($load_modus_old); # get 250 article which are the date of last_scan is very old (dump_scan)
#old
#article_last_live_scan(); # get all article from last live scan, where the script found errors
# very long in many languages (maybe later)
# replace with done articles
#article_with_error_from_dump_scan(); # get all articles error from the last dump scan
# replace with article_with_error_from_dump_scan2
#article_with_error_from_dump_scan2() if ($load_modus_dump); # get 250 articles of each error from the last dump scan,
#geo_error_article(); # get all articles with geo errors last days
# sort all articles (new + live)
@live_article = sort(@live_article);
# delet all double/multi input article
$number_of_live_tests = @live_article;
#print $number_of_live_tests."\n";
my @new_live_article;
my @split_line;
my @split_line_old;
if ($number_of_live_tests > 0) {
my $old_title = '';
my $all_errors_of_this_article = '';
my $i = -1;
foreach (@live_article) {
@split_line_old = @split_line;
@split_line = split(/\t/, $_);
my $current_title = $split_line[0];
$split_line[1] =~ s/\n//;
#print $current_title."\n";
my $number_of_split_line = @split_line;
if ($number_of_split_line != 2) {
print 'Problem with input line:'."\n";
print $_."\n";
die;
};
if ($old_title ne $current_title
and $old_title ne ''){
#save old
$i = $i+1;
$new_live_article[$i] = $old_title."\t".$all_errors_of_this_article;
$all_errors_of_this_article = '';
#print "result:".$new_live_article[$i]."\n";
}
# check new
if ($old_title eq $current_title) {
#double
$all_errors_of_this_article = $all_errors_of_this_article.', '.$split_line[1];
#print 'double: '.$current_title."\t".$all_errors_of_this_article."\n";
} else {
$all_errors_of_this_article = $split_line[1];
#print 'normal: '.$current_title."\t".$all_errors_of_this_article."\n";
}
$old_title = $current_title;
}
#save last
$i = $i+1;
$new_live_article[$i] = $old_title."\t".$all_errors_of_this_article;
@live_article = @new_live_article;
$number_of_live_tests = @live_article;
}
two_column_display('all articles without double:', $number_of_live_tests);
print LOGFILE 'articles without double'."\t".$number_of_live_tests."\n" if (!$starter_modus);
@new_live_article = (); # free memory
@split_line = (); # free memory
#foreach (@live_article) {
# print LOGFILE $_."\n";
#}
#print LOGFILE 'END LIST'."\n\n";
if ($number_of_live_tests == 0) {
# if after this load in live_modus no article found, then end the scan
$quit_program = 'yes';
$quit_reason = $quit_reason. 'no article in scan list for live'."\n";
}
}
}
}
sub article_last_live_scan{
my $file_input_live = $output_directory.$project.'/'.$project.'_'.$error_list_filename;
#print $file_input_live."\n";
open(LIVE, "<$file_input_live");
@live_article = <LIVE>;
close (LIVE);
$number_of_live_tests = @live_article;
two_column_display('from file articles last scan:', $number_of_live_tests);
print LOGFILE 'articles last scan:'."\t".$number_of_live_tests."\n" if (!$starter_modus);
}
sub new_article{
my $new_counter = 0;
my $limit = $_[0];
# oldest not scanned article
# select distinct title from cw_new where scan_live = 0 and project = 'dewiki' and daytime >= (select daytime from cw_new where scan_live = 0 and project = 'dewiki' order by daytime limit 1) order by daytime limit 250;
my $sql_text = "select distinct title from cw_new where scan_live = 0 and project = '".$project."' and daytime >= (select daytime from cw_new where scan_live = 0 and project = '".$project."' order by daytime limit 1) order by daytime limit ".$limit.";";
my $result = '';
my $sth = $dbh->prepare( $sql_text );
#print '<p class="smalltext"/>'.$sql_text."</p>\n";
$sth->execute;
while (my $arrayref = $sth->fetchrow_arrayref()) {
foreach(@$arrayref) {
$result = $_;
}
#print $result."\n";
push(@live_article, $result."\t".'0' );
$new_counter ++;
}
two_column_display('from db articles new:', $new_counter);
print LOGFILE 'articles new:'."\t\t".$new_counter. "\n" if (!$starter_modus);
$for_statistic_new_article = $new_counter;
}
sub last_change_article{
my $change_counter = 0;
my $limit = $_[0];
# oldest not scanned article
# select distinct title from cw_new where scan_live = 0 and project = 'dewiki' and daytime >= (select daytime from cw_new where scan_live = 0 and project = 'dewiki' order by daytime limit 1) order by daytime limit 250;
my $sql_text = "select distinct title from cw_change where scan_live = 0 and project = '".$project."' and daytime >= (select daytime from cw_change where scan_live = 0 and project = '".$project."' order by daytime limit 1) order by daytime limit ".$limit.";";
my $result = '';
my $sth = $dbh->prepare( $sql_text );
#print '<p class="smalltext"/>'.$sql_text."</p>\n";
$sth->execute;
while (my $arrayref = $sth->fetchrow_arrayref()) {
foreach(@$arrayref) {
$result = $_;
}
#print $result."\n";
push(@live_article, $result."\t".'0' );
$change_counter ++;
}
two_column_display('from db articles changed:', $change_counter);