From Wikisource
Jump to navigation Jump to search
#! /usr/bin/env python
# _*_ coding: utf8 _*_
# Remove soft redirects for specified months after fixing the pages linking to it
# Copyright © 2006—2010, GrafZahl ( user)
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# Uses some ideas from by Rob W.W. Hooft, Andre Engels, which is
# distributed under the terms of the MIT licence.
# run with standard args "-log -putthrottle:xx"
# Further arguments:
#	-cat:xxx
#		Specifies the category for which soft redirects should be
#		removed, for example: -cat:'Soft redirects/August 2006'
#		(replace the single quotes with whatever is appropriate for
#		your shell)
#	-dumplinks
#		Write all pages linking to a soft redirect page for the given
#		month to a file
#	-delete
#		Actually try to delete the pages (assumes sysop privileges!).
#		Otherwise the to-be-deleted page will be logged with
#		[to-be-deleted] prefix.
#	-xlink:xxx
#		Specifies a set of pages to be excluded from link correction
#		as a regular expression. For example, to exclude all
#		discussion archives, specify -xlink:'.*/Archive.*' (replace
#		the single quotes with whatever is appropriate for your
#		shell).
#	-nopipe:xxx
#		Specifies a set of soft redirects as a regular expression.
#		These redirects will not be added to corrected links as
#		pipes. Pipes that already exist will not be altered.

import catlib, re, sys, wikipedia


# Handle args

args = wikipedia.handleArgs()

month = False
delete = False
dumplinks = False
xlinks = []
nopipe = []

for arg in args:
	if arg[:5] == u'-cat:':
		month = arg[5:]
	elif arg == u'-delete':
		delete = True
	elif arg == u'-dumplinks':
		dumplinks = True
	elif arg[:7] == u'-xlink:':
		except re.error:
			wikipedia.output(u'(WWW) Ignoring invalid regular expression %s' % arg[7:])
	elif arg[:8] == u'-nopipe:':
		except re.error:
			wikipedia.output(u'(WWW) Ignoring invalid regular expression %s' % arg[8:])
		wikipedia.output(u'(WWW) Ignoring unrecognised argument: %s' % arg)

if not month:
	wikipedia.output(u'(FFF) No category given (-cat:xxx)')

# basic text tokens, etc.

cattitle = u'Category:%s' % month
base_redirover = u'#REDIRECT[[%s]]'
summ = u'[bot] shortcutting redirect(s)'
base_delsumm = u'[bot] deleting old soft redirect to [[%s]]'
comment_re = re.compile(r'(?ms)<!--.*?-->')
inconly_re = re.compile(r'(?ms)<includeonly>.*?</includeonly>')
nowiki_re = re.compile(r'(?ms)<nowiki>.*?</nowiki>')
link_re = re.compile(r'\[\[(?P<title>[^\]\|#]*)(?P<sectionlink>#[^\]\|]*)?(?P<pipe>\|[^\]]*)?\]\]')

# Function to count instances of a substring in a string, with possible overlap

def count_overlap(string, substring):
	count = 0
	start = string.find(substring) + 1
	while start:
		count += 1
		start = string.find(substring, start) + 1
	return count

def get_page(pagecache, site, title):
	page = wikipedia.Page(site, title)
		result = pagecache[page.title()]
	except KeyError:
		pagecache[page.title()] = page
		result = pagecache[page.title()]
	return result

# Function to extract all links to a given non-redirect page

def make_search_replace_list(pagetext, pagetitle, oldtitle, newtitle, dontpipe = False):
        """pagetext: Text to be searched
           pagetitle: Title of page to be searched (must not be a redirect page)
           oldtitle: title to be found, generated by wikipedia.Page.title()
           newtitle: New title to link to
           dontpipe: do not add pipe text if pipe is missing

           pagetitle, oldtitle and newtitle should be mutually different

           Returns list of (search, replace) tuples, where replace is, if
           possible, a relative link, if search is a relative link

           - Existing pipes are not altered
           - When no pipe exists, the old link will be used as pipes by
           - When no pipe exists and dontpipe == True, no pipe will be
        text = pagetext
        result = []
        # The following code is similar to wikipedia.Page.linkedPages
        ### Kill all comments, nowiki and includeonly
        text = re.sub(comment_re, r'', text)
        text = re.sub(nowiki_re, r'', text)
        text = re.sub(inconly_re, r'', text)
        ### Extract all links
        for match in link_re.finditer(text):
                # Extract title and calculate replacement if it is equivalent to newtitle
                oldlink =
                title ='title')
                sectionlink ='sectionlink')
                if sectionlink == None:
                        sectionlink = u''
                pipetext ='pipe')
                wtitle = title.strip()
		if len(wtitle) == 0: # Internal anchor
		# Check if the link begins with a colon
		if wtitle[0] == u':':
			colon = u':'
			colon = u''
                ### Ignore links to another wiki
                if site.isInterwikiLink(wtitle):
                ### Handle relative links
                relative = False
                nestlevel = count_overlap(wtitle, u'/../')
                if wtitle.startswith(u'../'):
                        relative = True
                        nestlevel += 1
                if (not wtitle.startswith(u'../' * nestlevel)) or (pagetitle.count(u'/') < nestlevel):
                        # not a valid link
                wpagetitle = pagetitle
                ##### Calculate absolute link
                for i in range(nestlevel):
                        wpagetitle = wpagetitle[:wpagetitle.rfind(u'/')]
                        wtitle = wtitle[3:]
                if relative:
                        wtitle = wpagetitle + u'/' + wtitle
                        # If the calculated title ends with /, it is stripped.
                        # Bug in MediaWiki?
                        if wtitle.endswith(u'/'):
                                wtitle = wtitle[:-1]
                if wtitle.startswith(u'/'):
                        wtitle = wpagetitle + wtitle
                        # Also a form of a relative link
                        relative = True
                ### Normalise title
                        wtitle = wikipedia.Page(site, wtitle).title()
                except wikipedia.Error:
                        # Something wrong with the title
                        wikipedia.output(u'(DDD) Title %s caused exception (pagetitle=%s, oldtitle=%s, newtitle=%s, oldlink=%s, extracted title=%s)' % (wtitle, pagetitle, oldtitle, newtitle, oldlink, title))
                if wtitle != oldtitle:
                        # It's some other link
                ### Replace link with new link
                wnewtitle = newtitle
                if relative:
			# Make it a relative link
                        ### How many levels are there in total in the page title?
                        totallevels = pagetitle.count(u'/') + 1
                        ### How many levels do the new title and the current page in common?
			##### Check '/' form first, otherwise count matching
			##### initial letters
			if wnewtitle.startswith(pagetitle):
				commonlevels = totallevels
                        	i = 0
                        	while wnewtitle[i] == pagetitle[i]:
                                	i += 1
				commonlevels = wnewtitle.count(u'/', 0, i + 1)
			### kill common levels from new title and add
			### sufficient "../"
                        for i in range(commonlevels):
                                wnewtitle = wnewtitle[wnewtitle.find(u'/') + 1:]
                        if commonlevels == totallevels:
                                wnewtitle = u'/' + wnewtitle
                        wnewtitle = (u'../' * (totallevels - commonlevels)) + wnewtitle
                if pipetext == None:
                        if dontpipe == False:
                                pipetext = u'|' + title
                                pipetext = u''
                newlink = u'[[%s%s%s%s]]' % ( colon, wnewtitle, sectionlink, pipetext )
                result.append((oldlink, newlink))
        return list(set(result))

# Start operation

site = wikipedia.getSite()
cat = catlib.Category(site, cattitle)
articles = list(cat.articles())

# Generate dictionary of texts linking to each soft redirect

pagecache = {}
linksdict = {}
included = set()
excluded = set()

for page in articles:
	refs = page.getReferences()
	linksdict[page.title()] = []
	for ref in refs:
		if ref.title() in excluded:
		do_include = True
		for xlink in xlinks:
			match = xlink.match(ref.title())
			if (match != None) and ( == ref.title()):
				do_include = False
		if do_include:

included = sorted(included)
excluded = sorted(excluded)

wikipedia.output(u'(III) The following pages will be link-corrected:')
for title in included:
	wikipedia.output(u'* [[%s]]' % title)

wikipedia.output(u'(III) The following pages will be EXCLUDED from link correction:')
for title in excluded:
	wikipedia.output(u'* [[%s]]' % title)

# Now check which links are deemed unpipeable

dontpipe = set()

for page in articles:
	for pattern in nopipe:
		match = pattern.match(page.title())
		if (match != None) and ( == page.title()):

dontpipe = sorted(dontpipe)

wikipedia.output(u'(III) The following old links will not have pipes added:')
for title in dontpipe:
	wikipedia.output(u'[[%s]]' % title)

# Dump links

if dumplinks:
	while True:
		fname = wikipedia.input(u'File name for list of pages linking to soft redirects?')
			f = file(fname, u'a')
			if f.tell() != 0:
				wikipedia.output(u'(EEE) File %s already exists. Please choose another file name.' % fname)
		except IOError:
			wikipedia.output(u'(EEE) IO Error during operation with %s. Please try again or choose another file name.' % fname)
	# Write links in Wiki markup. Exceptions terminate process.
	for title in linksdict.iterkeys():
		for link in linksdict[title]:
			f.write(u'* [[%s]] links to [[%s]]\n' % ( link, title ))
	wikipedia.output(u'Links written to file %s' % fname)

# Correct links without putting the corrected version at first and delete old pages

oldtextdict = {}
changedict = {}
backrefdict = {}
deleteset = set()

for title in linksdict.iterkeys():
	wikipedia.output(u'(III) Calculating updated links to %s' % title)
	softredir = get_page(pagecache, site, title)
	# Check if someone confused soft and hard redirs
	if softredir.isRedirectPage():
		wikipedia.output(u'(EEE) %s is a hard redirect, not a soft one' % title)
	# Extract new target
	newlist = softredir.linkedPages()
	### There should be only one normal link
	if len(newlist) != 1:
		wikipedia.output(u'(EEE) No unambiguous target for soft redirect %s' % title)
	new = newlist[0]
	newtitle = new.title()
	# HACK!
	if new.namespace() in (6, 14):
		newtitle = u':' + newtitle
	# End HACK
	redirover = base_redirover % newtitle
	# Correct links for each page individually
	for pagetitle in linksdict[title]:
		page = get_page(pagecache, site, pagetitle)
		# Back link
		if not backrefdict.has_key(pagetitle):
			backrefdict[pagetitle] = []
		# Special treatment for redirect pages
		# These can be fixed immediately because they have exactly one link
		if page.isRedirectPage():
				wikipedia.output(u'(XXX) Fake put: Overwriting [[%s]] with "%s", summary "%s"' % ( pagetitle, redirover, summ ))
			except wikipedia.Error:
				wikipedia.output(u'(EEE) Unable to edit redirect %s' % pagetitle)
		# get text tokens to be replaced with new link
				text = changedict[pagetitle]
			except KeyError:
				text = page.get()
				oldtextdict[pagetitle] = text
			if title in dontpipe:
				override_pipe = True
				override_pipe = False
			srlist = make_search_replace_list(text, pagetitle, title, newtitle, override_pipe)
			for sr in srlist:
				wikipedia.output(u'(XXX) Replacing "%s" with "%s" in [[%s]]' % ( sr[0], sr[1], pagetitle ))
				text = wikipedia.replaceExcept(text, re.escape(sr[0]), sr[1], ['comment', 'math', 'nowiki', 'pre'])
			changedict[pagetitle] = text
		except wikipedia.Error:
			wikipedia.output(u'(EEE) Unable to process %s' % pagetitle)
	# Add soft redirect page to the set of to-be-deleted pages
	if delete:
		wikipedia.output(u'(III) [to-be-deleted] %s' % title)

# Now update all links

for title in changedict.iterkeys():
	wikipedia.output('(III) Updating links in page %s' % title)
	# Get current version of page
	page = wikipedia.Page(site, title)
	# Update only if page wasn't edited since
		# Check if text has changed
		# Comparison of permalinks would be more efficient, but
		# unfortunately, pywikipedia's permalink feature is somewhat
		# broken
		if page.get() == oldtextdict[title]:
			wikipedia.output(u'(XXX) Fake put: Overwriting [[%s]] with summary "%s"' % ( title, summ ))
			wikipedia.output(u'(EEE) Not updating [[%s]]: Page was edited since' % title)
			# Don't delete soft redirects that still have issues
			for backlink in backrefdict[title]:
	except wikipedia.Error:
		wikipedia.output('(EEE) Unable to edit [[%s]]' % title)

# Lastly, delete the soft redirects

for title in deleteset:
		page = get_page(pagecache, site, title)
		delsumm = base_delsumm % page.linkedPages()[0].title()
		wikipedia.output(u'(XXX) Fake delete: [[%s]] with summary "%s"' % ( title, delsumm ))
	except wikipedia.Error:
		wikipedia.output(u'(EEE) Unable to delete %s' % title)
	except IndexError:
		wikipedia.output(u'(EEE) Not deleting %s: Unable to find redirect target' % title)