ViewVC Help

View File | Revision Log | View Changeset | Root Listing

root/Oni2/ValBot/Python/check_interwiki_links.py

Comparing ValBot/Python/check_interwiki_links.py (file contents):
Revision 1197 by iritscen, Fri Aug 29 03:52:17 2025 UTC vs.
Revision 1198 by iritscen, Sun Sep 21 21:50:56 2025 UTC

#	Line 1 | Line 1
1		# Check Interwiki Links
2		# by iritscen@yahoo.com
3	<	# Looks at each link on a page (or all the pages in a category) which uses a registered
4	<	# interwiki prefix and loads the linked page, verifying that it exists and that any section
5	<	# link, if present, is valid as well. The output will use the word "ERROR" when it cannot
6	<	# validate the interwiki link.
3	>	# Looks at each link on a page (or all the pages in a category) which uses a registered interwiki prefix and loads the linked page, verifying that it exists and that
4	>	# any section link, if present, is valid as well. The output will use the word "ERROR" when it cannot validate the interwiki link.
5		# Recommended viewing width:
6	<	# |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---|
6	>	# |---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ----|
7
8		import bs4
9		import pywikibot
#	Line 21 | Line 19 | from pywikibot.tools.formatter import co
19		from urllib.parse import urljoin
20
21		class IWLink:
22	<	def __init__(self, iw_prefix, prefix_url, full_url, page_name, page_slug, curl_response):
22	>	def __init__(self, iw_prefix, prefix_url, full_url, page_name, page_name_only, page_slug, hosting_page, curl_response):
23		self.iw_prefix = iw_prefix # e.g. "wp"
24		self.prefix_url = prefix_url # e.g. "https://en.wikipedia.org/wiki/"
25	<	self.full_url = full_url # e.g. "https://en.wikipedia.org/wiki/Easter_egg"
26	<	self.page_name = page_name # "Easter egg"
27	<	self.page_slug = page_slug # "Easter_egg"
25	>	self.full_url = full_url # e.g. "https://en.wikipedia.org/wiki/Marathon_(series)#Rampancy"
26	>	self.page_name = page_name # "Marathon (series)#Rampancy"
27	>	self.page_name_only = page_name # "Marathon (series)"
28	>	self.page_slug = page_slug # "Marathon_(series)#Rampancy"
29	>	self.hosting_page = hosting_page # "Easter eggs"; page where the link was found
30		self.curl_response = curl_response # a class defined in the Requests library
31
32		# Parallel arrays based on https://wiki.oni2.net/Special:Interwiki
#	Line 43 | Line 43 | unintended_redirects_found = 0
43		name_printed = 0
44
45		# Prints the name of a page on which something occurred, if it has not been printed before
46	<	def possibly_print(page_name):
46	>	def possibly_print(the_link):
47		global debug
48		global name_printed
49
50		if not name_printed and not debug:
51		pywikibot.stdout('')
52	<	pywikibot.stdout('From page "{}":'.format(page_name))
52	>	pywikibot.stdout('From page "{}":'.format(the_link.hosting_page))
53		name_printed = 1
54
55		# Search a page for the section specified in the link
#	Line 57 | Line 57 | def find_section(the_link, print_result)
57		global errors_issued
58
59		# Isolate section link
60	<	target_page_name, anchor_name = the_link.page_slug.split('#')
61	<	target_page_name_human = target_page_name.replace('_', ' ')
60	>	_, anchor_name = the_link.page_slug.split('#')
61
62		# Convert dot-notation hex entities to proper characters
63		replacements = [(r'\.22', '"'), (r'\.27', "'"), (r'\.28', '('), (r'\.29', ')')]
#	Line 79 | Line 78 | def find_section(the_link, print_result)
78
79		# Tell user what we found
80		if found_section == False:
81	<	possibly_print(the_link.page_name)
82	<	pywikibot.stdout('   ERROR: Could not find section "{0}" on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, target_page_name_human))
81	>	possibly_print(the_link)
82	>	pywikibot.stdout('   ERROR: Could not find section "{0}" on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, the_link.page_name))
83		errors_issued = errors_issued + 1
84		elif print_result == True:
85	<	pywikibot.stdout('   The section "{0}" was found on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, target_page_name_human))
85	>	pywikibot.stdout('   The section "{0}" was found on {1} page "{2}".'.format(anchor_name, the_link.iw_prefix, the_link.page_name))
86
87	<	# For a link that redirected us to another page, extract the name of the target page from
89	<	# the target page's source
87	>	# For a link that redirected us to another page, extract the name of the target page from the target page's source
88		def find_canonical_link(the_link):
89		# Extract link from this markup which contains name of redirected-to page:
90		# <link rel="canonical" href="https://en.wikipedia.org/wiki/Page_name"/>
#	Line 101 | Line 99 | def find_canonical_link(the_link):
99		else:
100		canonical_name = canonical_name[:tag_end]
101		if len(canonical_name) > 100:
102	<	# Certain things can cause the trim to fail; report error and avoid slamming the
105	<	# output with massive page source from a failed trim
102	>	# Certain things can cause the trim to fail; report error and avoid slamming the output with massive page source from a failed trim
103		pywikibot.stdout('   ERROR: The {0} link "{1}" is a redirect to "{2}…" (string overflow).'.format(the_link.iw_prefix, the_link.page_slug, canonical_name[:100]))
104		errors_issued = errors_issued + 1
105		else:
106	<	canonical_name = canonical_name.replace('_', ' ')
106	>	the_link.page_name = canonical_name.replace('_', ' ')
107		if '#' in the_link.page_slug:
108	<	_, anchor_name = the_link.page_slug.split('#')
109	<	pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}#{3}", which is a valid page. Checking for section on that page….'.format(the_link.iw_prefix, the_link.page_slug, canonical_name, anchor_name))
113	<	the_link.page_slug = the_link.page_slug.replace(the_link.page_name, canonical_name) # update page slug so that find_section() uses the right page name in its messages
108	>	the_link.page_name_only, _ = the_link.page_slug.split('#')
109	>	pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}", which is a valid page. Checking for section on that page….'.format(the_link.iw_prefix, the_link.page_name_only, the_link.page_name))
110		find_section(the_link, True)
111		else:
112	<	pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}", which is a valid page.'.format(the_link.iw_prefix, the_link.page_slug, canonical_name))
112	>	pywikibot.stdout('   The {0} link "{1}" is a redirect to "{2}", which is a valid page.'.format(the_link.iw_prefix, the_link.page_slug, the_link.page_name))
113
114		# Test an interwiki link and look for a section link if applicable
115		def test_interwiki_link(the_link):
#	Line 122 | Line 118 | def test_interwiki_link(the_link):
118
119		the_link.curl_response = fetch(the_link.full_url)
120
121	<	# One way we tell that a redirect occurred is by checking fetch's history, as it
122	<	# automatically follows redirects. This will catch formal redirects which come from pages
127	<	# such as Special:PermanentLink.
121	>	# One way we tell that a redirect occurred is by checking fetch's history, as it automatically follows redirects. This will catch formal redirects which come from
122	>	# pages such as Special:PermanentLink.
123		if the_link.curl_response.history != []:
124	<	possibly_print(the_link.page_name)
124	>	possibly_print(the_link)
125
126		# If linked page is in all caps, e.g. WP:BEANS, it's likely a deliberate use of a redirect
127		if the_link.page_slug.startswith('WP:') and the_link.page_slug == the_link.page_slug.upper():
#	Line 143 | Line 138 | def test_interwiki_link(the_link):
138		pywikibot.stdout('   ERROR: Unrecognized type of redirection (code "{0}") for {1} link "{2}". You should check the link manually.'.format(the_link.curl_response.history[0], the_link.iw_prefix, the_link.page_slug))
139		errors_issued = errors_issued + 1
140		elif the_link.curl_response.status_code != 200:
141	<	possibly_print(the_link.page_name)
141	>	possibly_print(the_link)
142		pywikibot.stdout('   ERROR: Got response code {0} for {1} link "{2}". The page may not exist.'.format(the_link.curl_response.status_code, the_link.iw_prefix, the_link.page_slug))
143		errors_issued = errors_issued + 1
144	<	# However the usual way that a redirect occurs is that MediaWiki redirects us sneakily
145	<	# using JavaScript, while returning code OK 200 as if the link was correct; this happens
146	<	# when a redirect page is accessed. We must detect these soft redirects by looking at the
152	<	# page source to find the redirect note inserted at the top of the page for the reader.
144	>	# However the usual way that a redirect occurs is that MediaWiki redirects us sneakily using JavaScript, while returning code OK 200 as if the link was correct; this
145	>	# happens when a redirect page is accessed. We must detect these soft redirects by looking at the page source to find the redirect note inserted at the top of the
146	>	# page for the reader.
147		elif 'Redirected from <a' in the_link.curl_response.text:
148		unintended_redirects_found = unintended_redirects_found + 1
149	<	possibly_print(the_link.page_name)
149	>	possibly_print(the_link)
150		pywikibot.stdout('   WARNING: Got silently redirected by {0} link "{1}". Checking the target page….'.format(the_link.iw_prefix, the_link.page_slug))
151	<	find_canonical_link(the_link)
151	>	find_canonical_link(the_link) # calls find_section() at end
152		elif '#' in the_link.page_slug:
153		find_section(the_link, False)
154
#	Line 172 | Line 166 | def scan_for_interwiki_links(page_text,
166		# Isolate strings that start with "[[prefix:" and end with "|" or "]"
167		iw_link = r"\[\[" + prefix + r":[^|\]]*(\||\])"
168		for match in re.finditer(iw_link, page_text):
169	<	the_link = IWLink(prefix, interwiki_urls[cur_prefix], "", page_name, "", "")
169	>	the_link = IWLink(prefix, interwiki_urls[cur_prefix], "", "", "", "", page_name, "")
170
171		# Extract just the page title from this regex match
172		s = match.start() + 2 + len(the_link.iw_prefix) + 1
173		e = match.end() - 1
174
175	<	# Commonly we use spaces instead of underscores, so fix that before querying
175	>	# Use underscores in slug used to constructed URL, but retain spaces for printable name
176		the_link.page_slug = page_text[s:e].replace(' ', '_')
177	<
178	<	# But use spaces for title when printing it
185	<	page_title_human = the_link.page_slug.replace('_', ' ')
186	<	if debug: pywikibot.stdout('   Validating {0} link "{1}"'.format(the_link.iw_prefix, page_title_human))
177	>	the_link.page_name = page_text[s:e]
178	>	if debug: pywikibot.stdout('   Validating {0} link "{1}"'.format(the_link.iw_prefix, the_link.page_name))
179		iw_found = iw_found + 1
180
181		# Construct full URL for the particular wiki

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines (old)
>	Changed lines (new)