wikispaces.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# Copyright (C) 2018 WikiTeam developers
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Documentation for users: https://github.com/WikiTeam/wikiteam/wiki
# Documentation for developers: http://wikiteam.readthedocs.com

import csv
import datetime
import os
import random
import re
import subprocess
import sys
import time
import urllib.request
#from internetarchive import get_item

# Requirements:
# zip command (apt-get install zip)
# ia command (pip install internetarchive, and configured properly)

"""
# You need a file with access and secret keys, in two different lines
iakeysfilename = '%s/.iakeys' % (os.path.expanduser('~'))
if os.path.exists(iakeysfilename):
    accesskey = open(iakeysfilename, 'r').readlines()[0].strip()
    secretkey = open(iakeysfilename, 'r').readlines()[1].strip()
else:
    print('Error, no %s file with S3 keys for Internet Archive account' % (iakeysfilename))
    sys.exit()
"""

def saveURL(wikidomain='', url='', filename='', path='', overwrite=False, iteration=1):
    filename2 = '%s/%s' % (wikidomain, filename)
    if path:
        filename2 = '%s/%s/%s' % (wikidomain, path, filename)
    if os.path.exists(filename2):
        if not overwrite:
            print('Warning: file exists on disk. Skipping download. Force download with parameter --overwrite')
            return
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    urllib.request.install_opener(opener)
    try:
        urllib.request.urlretrieve(url, filename2)
    except:
        sleep = 10 # seconds
        maxsleep = 30
        while sleep <= maxsleep:
            try:
                print('Error while retrieving: %s' % (url))
                print('Retry in %s seconds...' % (sleep))
                time.sleep(sleep)
                urllib.request.urlretrieve(url, filename2)
                return
            except:
                sleep = sleep * 2
        print('Download failed')
    
    #sometimes wikispaces returns invalid data, redownload in that cases
    #only 'pages'. 'files' binaries are a pain to open and check
    if (os.path.exists(filename2) and 'pages' in path) or \
        (os.path.exists(filename2) and path == '' and filename2.split('.')[-1] in ['xml', 'html', 'csv']):
        sleep2 = 60 * iteration
        raw = ''
        try:
            with open(filename2, 'r', encoding='utf-8') as f:
                raw = f.read()
        except:
            with open(filename2, 'r', encoding='latin-1') as f:
                raw = f.read()
        if re.findall(r'(?im)<title>TES and THE Status</title>', raw):
            print('Warning: invalid content. Waiting %d seconds and re-downloading' % (sleep2))
            time.sleep(sleep2)
            saveURL(wikidomain=wikidomain, url=url, filename=filename, path=path, overwrite=overwrite, iteration=iteration+1)

def undoHTMLEntities(text=''):
    """ Undo some HTML codes """

    # i guess only < > & " ' need conversion
    # http://www.w3schools.com/html/html_entities.asp
    text = re.sub('&lt;', '<', text)
    text = re.sub('&gt;', '>', text)
    text = re.sub('&amp;', '&', text)
    text = re.sub('&quot;', '"', text)
    text = re.sub('&#039;', '\'', text)

    return text

def convertHTML2Wikitext(wikidomain='', filename='', path=''):
    wikitext = ''
    wikitextfile = '%s/%s/%s' % (wikidomain, path, filename)
    if not os.path.exists(wikitextfile):
        print('Error retrieving wikitext, page is a redirect probably')
        return
    with open(wikitextfile, 'r') as f:
        wikitext = f.read()
    with open(wikitextfile, 'w') as f:
        m = re.findall(r'(?im)<div class="WikispacesContent WikispacesBs3">\s*<pre>', wikitext)
        if m:
            try:
                wikitext = wikitext.split(m[0])[1].split('</pre>')[0].strip()
                wikitext = undoHTMLEntities(text=wikitext)
            except:
                pass
        f.write(wikitext)

def downloadPage(wikidomain='', wikiurl='', pagename='', overwrite=False):
    pagenameplus = re.sub(' ', '+', pagename)
    pagename_ = urllib.parse.quote(pagename)
    
    #page current revision (html & wikitext)
    pageurl = '%s/%s' % (wikiurl, pagename_)
    filename = '%s.html' % (pagenameplus)
    print('Downloading page: %s' % (filename))
    saveURL(wikidomain=wikidomain, url=pageurl, filename=filename, path='pages', overwrite=overwrite)
    pageurl2 = '%s/page/code/%s' % (wikiurl, pagename_)
    filename2 = '%s.wikitext' % (pagenameplus)
    print('Downloading page: %s' % (filename2))
    saveURL(wikidomain=wikidomain, url=pageurl2, filename=filename2, path='pages', overwrite=overwrite)
    convertHTML2Wikitext(wikidomain=wikidomain, filename=filename2, path='pages')
    
    #csv with page history
    csvurl = '%s/page/history/%s?utable=WikiTablePageHistoryList&ut_csv=1' % (wikiurl, pagename_)
    csvfilename = '%s.history.csv' % (pagenameplus)
    print('Downloading page: %s' % (csvfilename))
    saveURL(wikidomain=wikidomain, url=csvurl, filename=csvfilename, path='pages', overwrite=overwrite)

def downloadFile(wikidomain='', wikiurl='', filename='', overwrite=False):
    filenameplus = re.sub(' ', '+', filename)
    filename_ = urllib.parse.quote(filename)
    
    #file full resolution
    fileurl = '%s/file/view/%s' % (wikiurl, filename_)
    filename = filenameplus
    print('Downloading file: %s' % (filename))
    saveURL(wikidomain=wikidomain, url=fileurl, filename=filename, path='files', overwrite=overwrite)
    
    #csv with file history
    csvurl = '%s/file/detail/%s?utable=WikiTablePageList&ut_csv=1' % (wikiurl, filename_)
    csvfilename = '%s.history.csv' % (filenameplus)
    print('Downloading file: %s' % (csvfilename))
    saveURL(wikidomain=wikidomain, url=csvurl, filename=csvfilename, path='files', overwrite=overwrite)

def downloadPagesAndFiles(wikidomain='', wikiurl='', overwrite=False):
    print('Downloading Pages and Files from %s' % (wikiurl))
    #csv all pages and files
    csvurl = '%s/space/content?utable=WikiTablePageList&ut_csv=1' % (wikiurl)
    saveURL(wikidomain=wikidomain, url=csvurl, filename='pages-and-files.csv', path='')
    #download every page and file
    totallines = 0
    with open('%s/pages-and-files.csv' % (wikidomain), 'r') as f:
        totallines = len(f.read().splitlines()) - 1
    with open('%s/pages-and-files.csv' % (wikidomain), 'r') as csvfile:
        filesc = 0
        pagesc = 0
        print('This wiki has %d pages and files' % (totallines))
        rows = csv.reader(csvfile, delimiter=',', quotechar='"')
        for row in rows:
            if row[0] == 'file':
                filesc += 1
                filename = row[1]
                downloadFile(wikidomain=wikidomain, wikiurl=wikiurl, filename=filename, overwrite=overwrite)
            elif row[0] == 'page':
                pagesc += 1
                pagename = row[1]
                downloadPage(wikidomain=wikidomain, wikiurl=wikiurl, pagename=pagename, overwrite=overwrite)
            if (filesc + pagesc) % 10 == 0:
                print('  Progress: %d of %d' % ((filesc + pagesc), totallines))
        print('  Progress: %d of %d' % ((filesc + pagesc), totallines))
    print('Downloaded %d pages' % (pagesc))
    print('Downloaded %d files' % (filesc))

def downloadSitemap(wikidomain='', wikiurl='', overwrite=False):
    print('Downloading sitemap.xml')
    saveURL(wikidomain=wikidomain, url=wikiurl, filename='sitemap.xml', path='', overwrite=overwrite)

def downloadMainPage(wikidomain='', wikiurl='', overwrite=False):
    print('Downloading index.html')
    saveURL(wikidomain=wikidomain, url=wikiurl, filename='index.html', path='', overwrite=overwrite)

def downloadLogo(wikidomain='', wikiurl='', overwrite=False):
    index = '%s/index.html' % (wikidomain)
    if os.path.exists(index):
        raw = ''
        try:
            with open(index, 'r', encoding='utf-8') as f:
                raw = f.read()
        except:
            with open(index, 'r', encoding='latin-1') as f:
                raw = f.read()
        m = re.findall(r'class="WikiLogo WikiElement"><img src="([^<> "]+?)"', raw)
        if m:
            logourl = m[0]
            logofilename = logourl.split('/')[-1]
            print('Downloading logo')
            saveURL(wikidomain=wikidomain, url=logourl, filename=logofilename, path='', overwrite=overwrite)
            return logofilename
    return ''

def printhelp():
    helptext = """This script downloads (and uploads) WikiSpaces wikis.

Parameters available:

--upload: upload compressed file with downloaded wiki
--admin: add item to WikiTeam collection (if you are an admin in that collection)
--overwrite: download again even if files exists locally
--overwrite-ia: upload again to Internet Archive even if item exists there
--help: prints this help text

Examples:

python3 wikispaces.py https://mywiki.wikispaces.com
   It downloads that wiki

python3 wikispaces.py wikis.txt
   It downloads a list of wikis (file format is a URL per line)

python3 wikispaces.py https://mywiki.wikispaces.com --upload
   It downloads that wiki, compress it and uploading to Internet Archive
"""
    print(helptext)
    sys.exit()

def duckduckgo():
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    urllib.request.install_opener(opener)
    
    wikis = []
    ignorewikis = [
        'https://wikispaces.com', 
        'https://www.wikispaces.com', 
        'https://wikispaces.net', 
        'https://www.wikispaces.net', 
    ]
    for i in range(1, 100000):
        url = 'https://duckduckgo.com/html/?q=%s%%20%s%%20site:wikispaces.com' % (random.randint(100, 5000), random.randint(1000, 9999))
        print('URL search', url)
        try:
            html = urllib.request.urlopen(url).read().decode('utf-8')
        except:
            print('Search error')
            time.sleep(30)
            continue
        html = urllib.parse.unquote(html)
        m = re.findall(r'://([^/]+?\.wikispaces\.com)', html)
        for wiki in m:
            wiki = 'https://' + wiki
            wiki = re.sub(r'https://www\.', 'https://', wiki)
            if not wiki in wikis and not wiki in ignorewikis:
                wikis.append(wiki)
                yield wiki
        sleep = random.randint(5,20)
        print('Sleeping %d seconds' % (sleep))
        time.sleep(sleep)

def main():
    upload = False
    isadmin = False
    overwrite = False
    overwriteia = False
    if len(sys.argv) < 2:
        printhelp()
    param = sys.argv[1]
    if not param:
        printhelp()
    if len(sys.argv) > 2:
        if '--upload' in sys.argv:
            upload = True
        if '--admin' in sys.argv:
            isadmin = True
        if '--overwrite' in sys.argv:
            overwrite = True
        if '--overwrite-ia' in sys.argv:
            overwriteia = True
        if '--help' in sys.argv:
            printhelp()
    
    wikilist = []
    if '://' in param:
        wikilist.append(param.rstrip('/'))
    elif param.lower() == 'duckduckgo':
        wikilist = duckduckgo()
        #for wiki in wikilist:
        #    print(wiki)
    else:
        with open(param, 'r') as f:
            wikilist = f.read().strip().splitlines()
            wikilist2 = []
            for wiki in wikilist:
                wikilist2.append(wiki.rstrip('/'))
            wikilist = wikilist2
    
    for wikiurl in wikilist:
        wikidomain = wikiurl.split('://')[1].split('/')[0]
        print('\n')
        print('#'*40,'\n Downloading:', wikiurl)
        print('#'*40,'\n')
        
        if upload and not overwriteia:
            itemid = 'wiki-%s' % (wikidomain)
            try:
                iahtml = ''
                try:
                    iahtml = urllib.request.urlopen('https://archive.org/details/%s' % (itemid)).read().decode('utf-8')
                except:
                    time.sleep(10)
                    iahtml = urllib.request.urlopen('https://archive.org/details/%s' % (itemid)).read().decode('utf-8')
                if iahtml and not re.findall(r'(?im)Item cannot be found', iahtml):
                    if not overwriteia:
                        print('Warning: item exists on Internet Archive. Skipping wiki. Force with parameter --overwrite-ia')
                        print('You can find it in https://archive.org/details/%s' % (itemid))
                        time.sleep(1)
                        continue
            except:
                pass
        
        dirfiles = '%s/files' % (wikidomain)
        if not os.path.exists(dirfiles):
            print('Creating directory %s' % (dirfiles))
            os.makedirs(dirfiles)
        dirpages = '%s/pages' % (wikidomain)
        if not os.path.exists(dirpages):
            print('Creating directory %s' % (dirpages))
            os.makedirs(dirpages)
        sitemapurl = 'https://%s/sitemap.xml' % (wikidomain)
        
        downloadSitemap(wikidomain=wikidomain, wikiurl=sitemapurl, overwrite=overwrite)
        if not os.path.exists('%s/sitemap.xml' % (wikidomain)):
            print('Error, wiki was probably deleted. Skiping wiki...')
            continue
        else:
            sitemapraw = ''
            try:
                with open('%s/sitemap.xml' % (wikidomain), encoding='utf-8') as g:
                    sitemapraw = g.read()
            except:
                with open('%s/sitemap.xml' % (wikidomain), encoding='latin-1') as g:
                    sitemapraw = g.read()
            if re.search(r'(?im)<h1>This wiki has been deactivated</h1>', sitemapraw):
                print('Error, wiki was deactivated. Skiping wiki...')
                continue
        
        downloadMainPage(wikidomain=wikidomain, wikiurl=wikiurl, overwrite=overwrite)
        if not os.path.exists('%s/index.html' % (wikidomain)):
            print('Error, wiki was probably deleted or expired. Skiping wiki...')
            continue
        else:
            indexraw = ''
            try:
                with open('%s/index.html' % (wikidomain), encoding='utf-8') as g:
                    indexraw = g.read()
            except:
                with open('%s/index.html' % (wikidomain), encoding='latin-1') as g:
                    indexraw = g.read()
            if re.search(r'(?im)<h1>Subscription Expired</h1>', indexraw):
                print('Error, wiki subscription expired. Skiping wiki...')
                continue
        
        downloadPagesAndFiles(wikidomain=wikidomain, wikiurl=wikiurl, overwrite=overwrite)
        logofilename = downloadLogo(wikidomain=wikidomain, wikiurl=wikiurl, overwrite=overwrite)
        
        if upload:
            itemid = 'wiki-%s' % (wikidomain)
            print('\nCompressing dump...')
            wikidir = wikidomain
            os.chdir(wikidir)
            print('Changed directory to', os.getcwd())
            wikizip = '%s.zip' % (wikidomain)
            subprocess.call('zip' + ' -r ../%s files/ pages/ index.html pages-and-files.csv sitemap.xml %s' % (wikizip, logofilename), shell=True)
            os.chdir('..')
            print('Changed directory to', os.getcwd())
            
            print('\nUploading to Internet Archive...')
            indexfilename = '%s/index.html' % (wikidir)
            if not os.path.exists(indexfilename):
                print('\nError dump incomplete, skipping upload\n')
                continue
            indexhtml = ''
            try:
                with open(indexfilename, 'r', encoding='utf-8') as f:
                    indexhtml = f.read()
            except:
                with open(indexfilename, 'r', encoding='latin-1') as f:
                    indexhtml = f.read()
            
            wikititle = ''
            try:
                wikititle = indexhtml.split('wiki: {')[1].split('}')[0].split("text: '")[1].split("',")[0].strip()
            except:
                wikititle = wikidomain
            if not wikititle:
                wikititle = wikidomain
            wikititle = wikititle.replace("\\'", " ")
            wikititle = wikititle.replace('\\"', " ")
            itemtitle = 'Wiki - %s' % wikititle
            itemdesc = '<a href=\"%s\">%s</a> dumped with <a href=\"https://github.com/WikiTeam/wikiteam\" rel=\"nofollow\">WikiTeam</a> tools.' % (wikiurl, wikititle)
            itemtags = ['wiki', 'wikiteam', 'wikispaces', wikititle, wikidomain.split('.wikispaces.com')[0], wikidomain]
            itemoriginalurl = wikiurl
            itemlicenseurl = ''
            m = ''
            try:
                m = re.findall(r'<a rel="license" href="([^<>]+?)">', indexhtml.split('<div class="WikiLicense')[1].split('</div>')[0])
            except:
                m = ''
            if m:
                itemlicenseurl = m[0]
            if not itemlicenseurl:
                itemtags.append('unknowncopyright')
            itemtags_ = ' '.join(["--metadata='subject:%s'" % (tag) for tag in itemtags])
            itemcollection = isadmin and 'wikiteam' or 'opensource'
            itemlang = 'Unknown'
            itemdate = datetime.datetime.now().strftime("%Y-%m-%d")
            itemlogo = logofilename and '%s/%s' % (wikidir, logofilename) or ''                
            callplain = "ia upload %s %s %s --metadata='mediatype:web' --metadata='collection:%s' --metadata='title:%s' --metadata='description:%s' --metadata='language:%s' --metadata='last-updated-date:%s' --metadata='originalurl:%s' %s %s" % (itemid, wikizip, itemlogo and itemlogo or '', itemcollection, itemtitle, itemdesc, itemlang, itemdate, itemoriginalurl, itemlicenseurl and "--metadata='licenseurl:%s'" % (itemlicenseurl) or '', itemtags_)
            print(callplain)
            subprocess.call(callplain, shell=True)
            
            """
            md = {
                'mediatype': 'web',
                'collection': itemcollection,
                'title': itemtitle,
                'description': itemdesc,
                'language': itemlang,
                'last-updated-date': itemdate,
                'subject': '; '.join(itemtags), 
                'licenseurl': itemlicenseurl,
                'originalurl': itemoriginalurl,
            }
            item = get_item(itemid)
            item.upload(wikizip, metadata=md, access_key=accesskey, secret_key=secretkey, verbose=True, queue_derive=False)
            item.modify_metadata(md)
            if itemlogo:
                item.upload(itemlogo, access_key=accesskey, secret_key=secretkey, verbose=True)
            """
            
            print('You can find it in https://archive.org/details/%s' % (itemid))
            os.remove(wikizip)

if __name__ == "__main__":
    main()