ViewVC Help

View File | Revision Log | View Changeset | Root Listing

root/Oni2/ValBot/Python/check_intrawiki_section_links.py

Comparing ValBot/Python/check_intrawiki_section_links.py (file contents):
Revision 1179 by iritscen, Fri Apr 28 00:53:24 2023 UTC vs.
Revision 1194 by iritscen, Mon Nov 18 04:00:08 2024 UTC

#	Line 26 | Line 26 | intrawiki_prefixes = ('Image', 'Special'
26		# URL for main namespace of our wiki
27		onigalore_url = 'https://wiki.oni2.net/'
28
29	<	# Tuple of interwiki prefixes, for passing over such links
29	>	# Tuple of interwiki prefixes, for recognizing and passing over such links
30		interwiki_prefixes = ('acronym', 'cache', 'commons', 'dictionary', 'google', 'metawikimedia', 'mw', 'wikibooks', 'wikidata', 'wikimedia', 'wikinews', 'wikipedia', 'wikiquote', 'wikisource', 'wikispecies', 'wikiversity', 'wikivoyage', 'wikt', 'wiktionary', 'wp')
31
32		# List of chapter names, for substitution into links that use "{{Cn}}" transclusion
#	Line 35 | Line 35 | chapter_names = ['CHAPTER_00_._COMBAT_TR
35		# Tuple of patterns for recognizing wikilinks
36		# Pattern 1: Detect "[[anything]]", "[[any:thing]]", "[[any|thing]]", "[[any:thi|ng]]"
37		# Pattern 2: Detect "{{SectionLink|Page|Section name}}", "{{SectionLink||Section name}}"
38	<	link_patterns = ("\[\[[^|\]]*(\||\])", "\{\{SectionLink\|[^|\}]*\|[^|\}]*\}\}")
38	>	link_patterns = (r"\[\[[^|\]]*(\||\])", r"\{\{SectionLink\|[^|\}]*\|[^|\}]*\}\}")
39
40		# Initialize globals
41		debug = 0
#	Line 43 | Line 43 | pages_checked = 0
43		iw_found = 0
44		advice_issued = 0
45		errors_issued = 0
46	+	name_printed = 0
47	+
48	+	# Prints the name of a page on which something occurred, if it has not been printed before
49	+	def possibly_print(page_name):
50	+	global debug
51	+	global name_printed
52	+
53	+	if not name_printed and not debug:
54	+	pywikibot.stdout('')
55	+	pywikibot.stdout('From page "{}":'.format(page_name))
56	+	name_printed = 1
57	+
58	+	# Search a page for the section specified in the link
59	+	def find_section(page_text, page_name, page_slug, print_result):
60	+	global errors_issued
61	+	found_section = False
62	+
63	+	# Isolate section link or text fragment link
64	+	target_page_name, anchor_name = page_slug.split('#', 1)
65	+	target_page_name_human = target_page_name.replace('_', ' ')
66	+
67	+	# First check if this is a text fragment directive, and look for it if so
68	+	if anchor_name.startswith(':~:text='):
69	+	if debug: pywikibot.stdout('         Found text fragment directive {} from URL {}.'.format(anchor_name, page_slug))
70	+	anchor_name = anchor_name[8:]
71	+	# We're only checking the first text directive, so strip add'l ones if present
72	+	addl_fragment = anchor_name.find('&text=')
73	+	if addl_fragment != -1:
74	+	anchor_name = anchor_name[:addl_fragment]
75	+	search_terms = anchor_name.split(',')
76	+	# Delete prefix and suffix terms because they aren't needed
77	+	if search_terms[0].endswith('-'):
78	+	search_terms.pop(0)
79	+	if search_terms[-1].startswith('-'):
80	+	search_terms.pop()
81	+	# Remake text directive with the terms separated by spaces as they should be in the page text
82	+	newSep = ' '
83	+	search_string = newSep.join(search_terms)
84	+	if debug: pywikibot.stdout('         Converted text fragment to string "{}".'.format(search_string))
85	+	if search_string in page_text:
86	+	found_section = True
87	+	if debug and not print_result: pywikibot.stdout('         Found text fragment!')
88	+
89	+	# If we're still here, it's a section link; read linked page to see if it really has this
90	+	# anchor link
91	+	if found_section == False:
92	+	if debug: pywikibot.stdout('         Searching for section link {} on page.'.format(anchor_name))
93	+	soup = BeautifulSoup(page_text, 'html.parser')
94	+	# Search for a span with this ID
95	+	for span_tag in soup.findAll('span'):
96	+	span_name = span_tag.get('id', None)
97	+	if span_name == anchor_name:
98	+	if debug and not print_result: pywikibot.stdout('         Found section in a span!')
99	+	found_section = True
100	+	break
101	+	if found_section == False:
102	+	# Search for a div with this ID
103	+	for span_tag in soup.findAll('div'):
104	+	span_name = span_tag.get('id', None)
105	+	if span_name == anchor_name:
106	+	if debug and not print_result: pywikibot.stdout('         Found section in a div!')
107	+	found_section = True
108	+	break
109	+	if found_section == False:
110	+	possibly_print(page_name)
111	+	pywikibot.stdout('   ERROR: Could not find section "{0}" on page {1}!'.format(anchor_name, target_page_name_human))
112	+	errors_issued += 1
113	+	elif debug and print_result:
114	+	pywikibot.stdout('   The section "{0}" was found on page "{1}".'.format(anchor_name, target_page_name_human))
115	+
116	+	# For a link that redirected us to another page, extract the name of the target page from
117	+	# the target page's source
118	+	def find_canonical_link(page_text, page_name, page_slug):
119	+	# Extract link from this markup which contains name of redirected-to page:
120	+	# <link rel="canonical" href="https://en.wikipedia.org/wiki/Page_name"/>
121	+	# "wgPageName":"Namespace:Page_name",
122	+	canonical_name = page_text.split('"wgPageName":"')[-1]
123	+	tag_end = canonical_name.find('",')
124	+
125	+	if tag_end == -1:
126	+	pywikibot.stdout('   ERROR: The link "{}" is a redirect page, but this script could not isolate the target page name.'.format(page_slug))
127	+	errors_issued = errors_issued + 1
128	+	else:
129	+	canonical_name = canonical_name[:tag_end]
130	+	if len(canonical_name) > 100:
131	+	# Certain things can cause the trim to fail; report error and avoid slamming the
132	+	# output with massive page source from a failed trim
133	+	pywikibot.stdout('   ERROR: The link "{}" is a redirect to "{2}…" (string overflow).'.format(page_slug, canonical_name[:100]))
134	+	errors_issued = errors_issued + 1
135	+	else:
136	+	canonical_name = canonical_name.replace('_', ' ')
137	+	if '#' in page_slug:
138	+	_, anchor_name = page_slug.split('#')
139	+	if debug: pywikibot.stdout('   The link "{0}" is a redirect to "{1}#{2}", which is a valid page. Checking section link….'.format(page_slug, canonical_name, anchor_name))
140	+	find_section(page_text, page_name, page_slug, True)
141	+	else:
142	+	pywikibot.stdout('   The link "{0}" is a redirect to "{1}", which is a valid page.'.format(page_slug, canonical_name))
143	+
144	+	# Test an intrawiki link and look for a section link if applicable
145	+	def test_intrawiki_link(iw_url, page_name, page_slug):
146	+	global advice_issued
147	+	global errors_issued
148	+
149	+	response = fetch(iw_url)
150	+
151	+	# One way we tell that a redirect occurred is by checking fetch's history, as it
152	+	# automatically follows redirects. This will catch formal redirects which come from pages
153	+	# such as Special:PermanentLink.
154	+	if response.history != []:
155	+	permalink1 = 'Special:PermanentLink/'.lower()
156	+	permalink2 = 'Special:Permalink/'.lower()
157	+	page_slug_lower = page_slug.lower()
158	+	if page_slug_lower.startswith(permalink1) or page_slug_lower.startswith(permalink2):
159	+	if debug:
160	+	possibly_print(page_name)
161	+	pywikibot.stdout('   Got redirection code "{0}" for permanent revision link "{1}". Checking the target page….'.format(response.history[0], page_slug))
162	+	find_canonical_link(response.text, page_name, page_slug)
163	+	else:
164	+	possibly_print(page_name)
165	+	pywikibot.stdout('   ERROR: Unrecognized type of redirection (code "{0}") for link "{1}". You should check the link manually.'.format(response.history[0], page_slug))
166	+	advice_issued += 1
167	+	elif response.status_code != 200:
168	+	possibly_print(page_name)
169	+	pywikibot.stdout('   ERROR: Got response code {0} on URL {1}. The target page may not exist.'.format(response.status_code, iw_url))
170	+	errors_issued += 1
171	+	# However the usual way that a redirect occurs is that MediaWiki redirects us sneakily
172	+	# using JavaScript, while returning code OK 200 as if the link was correct; this happens
173	+	# when a redirect page is accessed. We must detect these soft redirects by looking at the
174	+	# page source to find the redirect note inserted at the top of the page for the reader.
175	+	elif 'Redirected from <a' in response.text:
176	+	if debug:
177	+	possibly_print(page_name)
178	+	pywikibot.stdout('   Got silently redirected by link "{}". Checking the target page….'.format(page_slug))
179	+	find_canonical_link(response.text, page_name, page_slug)
180	+	else: # URL is OK, so proceed
181	+	find_section(response.text, page_name, page_slug, False)
182
183		# Searches the given page text for intrawiki links with section links in them
184		def scan_for_intrawiki_links(page_text, page_name):
#	Line 51 | Line 187 | def scan_for_intrawiki_links(page_text,
187		global iw_found
188		global advice_issued
189		global errors_issued
190	+	global name_printed
191		pages_checked += 1
192		name_printed = 0
193
#	Line 74 | Line 211 | def scan_for_intrawiki_links(page_text,
211		target_end = 2 # "}}"
212		s = match.start() + target_start # remove the link-opening markup
213		e = match.end() - target_end # remove the link-ending markup
214	<	link_text = page_text[s:e]
214	>	page_slug = page_text[s:e]
215
216		# The second link type will look like "Page|Section" or "|Section", so fix that pipe
217		if i == 1:
218	<	link_text = link_text.replace('|', '#')
218	>	page_slug = page_slug.replace('|', '#')
219
220		# Sometimes we use a space char. instead of a '_', so fix that before querying
221	<	link_text = link_text.replace(' ', '_')
222	<	if debug: pywikibot.stdout('      Found link {0}.'.format(link_text))
221	>	page_slug = page_slug.replace(' ', '_')
222	>	if debug: pywikibot.stdout('      Found link {0}.'.format(page_slug))
223
224		# If this link doesn't have a section link in it, then we don't care about it, as
225		# MediaWiki takes care of checking basic intrawiki links
226	<	if not '#' in link_text:
226	>	if not '#' in page_slug:
227		if debug: pywikibot.stdout('         Link doesn\'t have a section anchor in it. Skipping.')
228		continue
229
#	Line 95 | Line 232 | def scan_for_intrawiki_links(page_text,
232		is_interwiki = False
233		if found_iw_match == False:
234		for prefix in interwiki_prefixes:
235	<	if prefix + ":" in link_text:
236	<	if debug: pywikibot.stdout('         Skipping link {} because it is an interwiki link.'.format(link_text))
235	>	if prefix + ":" in page_slug:
236	>	if debug: pywikibot.stdout('         Skipping link {} because it is an interwiki link.'.format(page_slug))
237		is_interwiki = True
238		break
239		if is_interwiki:
240		continue
241
242	<	# If there is a '{' in the link, then probably it's a link built on transcluded text
243	<	# like "Quotes/Diary#{{C3}}", which we cannot expand and work with, so skip it
244	<	if '{' in link_text:
242	>	# If there is a '{' in the link, then probably it's a link built on transcluded text.
243	>	# If it's a chapter template transclusion like "Quotes/Diary#{{C3}}", expand it using
244	>	# our "chapter_names" array. If it's another type of transclusion, punt it to the user.
245	>	if '{' in page_slug:
246		ch_link_pattern = re.compile(r"{{C[0-9]*}}")
247	<	ch_link = ch_link_pattern.search(link_text)
247	>	ch_link = ch_link_pattern.search(page_slug)
248		if debug: pywikibot.stdout('         Found transclusion in link: "{}".'.format(ch_link.group(0)))
249		if ch_link:
250		ch_link_match = ch_link.group(0)
#	Line 117 | Line 255 | def scan_for_intrawiki_links(page_text,
255		if ch_num_match >= 0 and ch_num_match <= 14:
256		ch_name = chapter_names[ch_num_match]
257		replace_pattern = re.compile(r"{{C" + ch_num.group(0) + r"}}")
258	<	link_text = replace_pattern.sub(ch_name, link_text)
259	<	if debug: pywikibot.stdout('         After performing transclusion, link is now "{}".'.format(link_text))
258	>	page_slug = replace_pattern.sub(ch_name, page_slug)
259	>	if debug: pywikibot.stdout('         After performing transclusion, link is now "{}".'.format(page_slug))
260		else:
261	<	if not name_printed and not debug:
262	<	pywikibot.stdout('From page "{}":'.format(page_name))
263	<	name_printed = 1
126	<	pywikibot.stdout('   ADVICE: Link {0} transcludes a chapter name using an out-of-range number, {1}.'.format(link_text, ch_num_match))
127	<	advice_issued += 1
261	>	possibly_print(page_name)
262	>	pywikibot.stdout('   ERROR: Link {0} transcludes a chapter name using an out-of-range number, {1}.'.format(page_slug, ch_num_match))
263	>	errors_issued += 1
264		continue
265		else:
266	<	if not name_printed and not debug:
267	<	pywikibot.stdout('From page "{}":'.format(page_name))
132	<	name_printed = 1
133	<	pywikibot.stdout('   ADVICE: Link {} seems to be transcluding a chapter name, but this script couldn\'t read it.'.format(link_text))
266	>	possibly_print(page_name)
267	>	pywikibot.stdout('   ADVICE: Link {} seems to be transcluding a chapter name, but this script couldn\'t read it.'.format(page_slug))
268		advice_issued += 1
269		continue
270		else:
271	<	if not name_printed and not debug:
272	<	pywikibot.stdout('From page "{}":'.format(page_name))
139	<	name_printed = 1
140	<	pywikibot.stdout('   ADVICE: Link {0} seems to use transclusion. This script can understand chapter name transclusions such as "{1}" but it doesn\'t recognize this one so it can\'t be verified. You should check the link manually.'.format(link_text, "{{C7}}"))
271	>	possibly_print(page_name)
272	>	pywikibot.stdout('   ADVICE: Link {0} seems to use transclusion. This script can understand chapter name transclusions such as "{1}" but it doesn\'t recognize this one so it can\'t be verified. You should check the link manually.'.format(page_slug, "{{C7}}"))
273		advice_issued += 1
274		continue
275
276		# If this is a relative "/" link, use the current page as the basis for the URL. Note
277		# that only a leading slash is looked for, so if there's multiple steps down ("/x/y"),
278		# we're out of luck.
279	<	if link_text.startswith('/'):
280	<	link_text = page_name + link_text
281	<	if debug: pywikibot.stdout('         Changed link_text to {} on account of "/".'.format(link_text))
279	>	if page_slug.startswith('/'):
280	>	page_slug = page_name + page_slug
281	>	if debug: pywikibot.stdout('         Changed page_slug to {} on account of "/".'.format(page_slug))
282
283		# If this is a relative "../" link, find the parent page, set ourselves to that page,
284		# then remove the relative portion of the link. Note that this is only performed once,
285		# so if there's multiple steps back ("../../"), we're out of luck.
286	<	if link_text.startswith('../'):
286	>	if page_slug.startswith('../'):
287		last_slash = page_name.rfind('/')
288		page_name2 = page_name[0:last_slash]
289		if debug: pywikibot.stdout('         Changed page_name to {} on account of "../".'.format(page_name2))
290	<	link_text = link_text[3:len(link_text)]
291	<	if debug: pywikibot.stdout('         Changed link_text to {} on account of "../".'.format(link_text))
290	>	page_slug = page_slug[3:len(page_slug)]
291	>	if debug: pywikibot.stdout('         Changed page_slug to {} on account of "../".'.format(page_slug))
292		# If this is now going to be a bare section link for the parent page, don't add a
293		# slash, otherwise do because we are drilling down to another subpage
294	<	if link_text.startswith('#'):
295	<	link_text = page_name2 + link_text
294	>	if page_slug.startswith('#'):
295	>	page_slug = page_name2 + page_slug
296		else:
297	<	link_text = page_name2 + '/' + link_text
297	>	page_slug = page_name2 + '/' + page_slug
298
299		# If this is a bare section link, build URL based on this page
300	<	if link_text.startswith('#'):
300	>	if page_slug.startswith('#'):
301		iw_url = onigalore_url + page_name2
302		iw_found += 1
303	<	if debug: pywikibot.stdout('         Found link to this very page, {}.'.format(link_text))
303	>	if debug: pywikibot.stdout('         Found link to this very page, {}.'.format(page_slug))
304		found_iw_match = True
305	<	link_text = page_name2 + link_text
305	>	page_slug = page_name2 + page_slug
306
307		# If there's no ":" in the link (before the section link, where a colon would just be
308		# part of the text) then it's a Main namespace article; proceed with building URL
309		if found_iw_match == False:
310	<	if not re.search(":.*#", link_text):
311	<	iw_url = onigalore_url + link_text
310	>	if not re.search(":.*#", page_slug):
311	>	iw_url = onigalore_url + page_slug
312		iw_found += 1
313		if debug: pywikibot.stdout('         Link is to a Main namespace page.')
314		found_iw_match = True
#	Line 185 | Line 317 | def scan_for_intrawiki_links(page_text,
317		# before building URL
318		if found_iw_match == False:
319		for prefix in intrawiki_prefixes:
320	<	if prefix + ":" in link_text:
321	<	iw_url = onigalore_url + link_text
320	>	if prefix + ":" in page_slug:
321	>	iw_url = onigalore_url + page_slug
322		if debug: pywikibot.stdout('         Identified namespace {}.'.format(prefix))
323		iw_found += 1
324		found_iw_match = True
#	Line 194 | Line 326 | def scan_for_intrawiki_links(page_text,
326
327		# If we still haven't turned this match into a URL, something's gone wrong
328		if (found_iw_match == False) or (iw_url == ""):
329	<	if not name_printed and not debug:
330	<	pywikibot.stdout('From page "{}":'.format(page_name))
199	<	name_printed = 1
200	<	pywikibot.stdout('   ERROR: Couldn\'t figure out link {}.'.format(link_text))
329	>	possibly_print(page_name)
330	>	pywikibot.stdout('   ERROR: Couldn\'t figure out link {}.'.format(page_slug))
331		continue
332
333		# Test the URL
334		iw_url = iw_url.replace(' ', '_')
335	<	if debug: pywikibot.stdout('         Reading page at {}...'.format(iw_url))
336	<	response = fetch(iw_url)
335	>	if debug: pywikibot.stdout('         Reading page at {}….'.format(iw_url))
336	>	test_intrawiki_link(iw_url, page_name, page_slug)
337
338	<	# Redirects are followed automatically by fetch() and treated as "200"s; the way we can
339	<	# tell that a redirect occurred is by checking fetch's history
210	<	if response.history != []:
211	<	if not name_printed and not debug:
212	<	pywikibot.stdout('From page "{}":'.format(page_name))
213	<	name_printed = 1
214	<	pywikibot.stdout('   ADVICE: Got redirection code ({0}) on URL "{1}". You should check the link manually.'.format(response.history[0], iw_url))
215	<	advice_issued += 1
216	<	elif response.status_code != 200:
217	<	if not name_printed and not debug:
218	<	pywikibot.stdout('From page "{}":'.format(page_name))
219	<	name_printed = 1
220	<	pywikibot.stdout('   ERROR: Got response code {0} on URL {1}. The target page may not exist.'.format(response.status_code, iw_url))
221	<	errors_issued += 1
222	<	else:
223	<	# Isolate section link
224	<	pre_section, section_name = link_text.split('#', 1)
225	<	if debug: pywikibot.stdout('         Searching for section link {} on page.'.format(section_name))
226	<
227	<	# Convert slash character to the dot-notation hex encoding that MediaWiki uses
228	<	section_name = section_name.replace('/', '.2F')
229	<
230	<	# Read linked page to see if it really has this anchor link
231	<	soup = BeautifulSoup(response.text, 'html.parser')
232	<	found_section = False
233	<	for span_tag in soup.findAll('span'):
234	<	span_name = span_tag.get('id', None)
235	<	if span_name == section_name:
236	<	if debug: pywikibot.stdout('         Found section!')
237	<	found_section = True
238	<	break
239	<	if found_section == False:
240	<	if not name_printed and not debug:
241	<	pywikibot.stdout('From page "{}":'.format(page_name))
242	<	name_printed = 1
243	<	pywikibot.stdout('   ERROR: Could not find section {0} on page {1}!'.format(section_name, pre_section))
244	<	errors_issued += 1
245	<
246	<	def main(*args):
247	<	global debug
338	>	# Print a wrap-up message
339	>	def print_summary():
340		global pages_checked
341		global iw_found
342		global advice_issued
343		global errors_issued
344	+
345	+	page_str = "pages"
346	+	if pages_checked == 1:
347	+	page_str = "page"
348	+
349	+	link_str = "links"
350	+	if iw_found == 1:
351	+	link_str = "link"
352	+
353	+	pywikibot.stdout('Checked {0} {1} and found {2} intrawiki {3}.'.format(pages_checked, page_str, iw_found, link_str))
354	+	pywikibot.stdout('While attempting to follow section links….')
355	+
356	+	if advice_issued == 0:
357	+	pywikibot.stdout('   No advice on potential problems was issued.')
358	+	elif advice_issued == 1:
359	+	pywikibot.stdout('   1 piece of advice on a potential problem was issued.')
360	+	else:
361	+	pywikibot.stdout('   {} pieces of advice on potential problems were issued.'.format(advice_issued))
362	+
363	+	error_str = "errors were"
364	+	if errors_issued == 1:
365	+	error_str = "error was"
366	+	pywikibot.stdout('   {0} {1} encountered.'.format(errors_issued, error_str))
367	+
368	+	# Main function
369	+	def main(*args):
370	+	global debug
371		search_cat = ''
372		search_page = ''
373
374	+	# Process arguments
375		local_args = pywikibot.handle_args(args)
256	–	genFactory = pagegenerators.GeneratorFactory()
257	–
376		for arg in local_args:
377		if arg.startswith('-cat:'):
378		search_cat = arg[5:]
#	Line 271 | Line 389 | def main(*args):
389		# This line of code enumerates the methods in the 'page' class
390		#pywikibot.stdout(format(dir(page)))
391
392	+	# Check specified page or loop through specified category and check all pages
393		if search_cat != '':
394		cat_obj = pywikibot.Category(site, search_cat)
395		generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True)
#	Line 282 | Line 401 | def main(*args):
401		if debug: pywikibot.stdout('Checking page {0}'.format(page.title()))
402		scan_for_intrawiki_links(page.text, page.title())
403
404	<	page_str = "pages"
405	<	if pages_checked == 1:
287	<	page_str = "page"
288	<
289	<	link_str = "links"
290	<	if iw_found == 1:
291	<	link_str = "link"
292	<
293	<	pywikibot.stdout('Checked {0} {1} and found {2} intrawiki {3}.'.format(pages_checked, page_str, iw_found, link_str))
294	<	pywikibot.stdout('While attempting to follow section links...')
295	<
296	<	if advice_issued == 0:
297	<	pywikibot.stdout('   No advice on potential problems was issued.')
298	<	elif advice_issued == 1:
299	<	pywikibot.stdout('   1 piece of advice on a potential problem was issued.')
300	<	else:
301	<	pywikibot.stdout('   {} pieces of advice on potential problems were issued.'.format(advice_issued))
302	<
303	<	error_str = "errors were"
304	<	if errors_issued == 1:
305	<	error_str = "error was"
306	<	pywikibot.stdout('   {0} {1} encountered.'.format(errors_issued, error_str))
404	>	# Print the results
405	>	print_summary()
406
407		if __name__ == '__main__':
408		main()

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines (old)
>	Changed lines (new)