User:Inductiveload/Scripts/Page concatenator

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#       untitled.py
#
#       Copyright 2010 Inductiveload
#
#       This program is free software; you can redistribute it and/or modify
#       it under the terms of the GNU General Public License as published by
#       the Free Software Foundation; either version 2 of the License, or
#       (at your option) any later version.
#
#       This program is distributed in the hope that it will be useful,
#       but WITHOUT ANY WARRANTY; without even the implied warranty of
#       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#       GNU General Public License for more details.
#
#       You should have received a copy of the GNU General Public License
#       along with this program; if not, write to the Free Software
#       Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#       MA 02110-1301, USA.

import pw_script_header
import wikipedia
import codecs
import re

FILELIST = "/home/john/src/pw/zz_gendata.txt"
DESTINATION = 'User:Inductiveload/Sandbox7'
OUTFILE = '/tmp/concatfile.txt'


def find_matching_braces(text, first_brace_index ):
    """finds the index of the matching right brace to a left brace"""

    lbrace = text[first_brace_index]

    if lbrace == '{':
        rbrace = '}'
    elif lbrace == '[':
        rbrace = ']'
    elif lbrace == '(':
        rbrace = ')'
    elif lbrace == '<':
        rbrace == '>'
    else:
        print "(ERR) not a brace"

    # brace counter
    count = 0
    char_number = first_brace_index

    for char in text[first_brace_index:]:
        if char == lbrace:
            count += 1
        elif char == rbrace:
            count -= 1
        elif count == 0:
            break

        char_number += 1

    if count != 0:
        print "(ERR) unbalanced brackets"
        return None
    else:
        return char_number

def find_header(wikitext):


    m = re.search(r'({{\s*header)', wikitext)

    if m:
        header_start = wikitext.find(m.group(1))
        print "(INF) Header found, starting at %d" % header_start

    else:
        print "(ERR) Header not found."
        return 0, 0

    header_end = find_matching_braces(wikitext, header_start)
    print "(INF) header ends at char: %d" % header_end

    header = wikitext[header_start:header_end]
    #print "\n", header, "\n"

    return header_start, header_end

def main():


    filelist = codecs.open(FILELIST, 'r', 'utf-8')
    site = wikipedia.getSite()

    tempfile = codecs.open('/tmp/wikt.txt', 'w', 'utf-8')


    text = ''
    for line in filelist:

        print "(INF) processing page: %s" % line.strip()

        page = wikipedia.Page(site, line.strip())
        newtext = page.get()

        start, end = find_header(newtext)

        newtext = newtext[end:]

        newtext = re.sub("===", "", newtext)
        newtext = re.sub("==", "", newtext)

        text += newtext

    tempfile.write(text)
    tempfile.close()
    
    
    print "(INF) Saving text to %s" % OUTFILE
    outfile = codecs.open(OUTFILE, 'w', 'utf-8')
    outfile.write(text)
    outfile.close()
    
    print "(INF) Uploading concatenated pages to %s." % DESTINATION
    cont = raw_input("continue? [y/n]")

    if cont == 'y':
        page = wikipedia.Page(site, DESTINATION)
        page.put(text, 'Uploading concatenated pages to %s.'% DESTINATION, minorEdit=False)

    return 0

if __name__ == '__main__':
    main()