', re.IGNORECASE)
#offset = 0
#while True:
#wikipedia.output("Google: Querying page %d" % (offset / 100 + 1))
#address = "http://www.google.com/search?q=%s&num=100&hl=en&start=%d" % (urllib.quote_plus(query), offset)
## we fake being Firefox because Google blocks unknown browsers
#request = urllib2.Request(address, None, {'User-Agent': 'Mozilla/5.0 (X11; U; Linux i686; de; rv:1.8) Gecko/20051128 SUSE/1.5-0.1 Firefox/1.5'})
#urlfile = urllib2.urlopen(request)
#page = urlfile.read()
#urlfile.close()
#for url in linkR.findall(page):
#yield url
#if "" in page: # Is there a "Next" link for next page of results?
#offset += 100 # Yes, go to next page of results.
#else:
#return
#########
def __iter__(self):
site = wikipedia.getSite()
# restrict query to local site
localQuery = '%s site:%s' % (self.query, site.hostname())
base = 'http://%s%s' % (site.hostname(), site.nice_get_address(''))
for url in self.queryGoogle(localQuery):
if url[:len(base)] == base:
title = url[len(base):]
page = wikipedia.Page(site, title)
yield page
def MySQLPageGenerator(query):
'''
'''
import MySQLdb as mysqldb
site = wikipedia.getSite()
conn = mysqldb.connect(config.db_hostname, db = site.dbName(),
user = config.db_username,
passwd = config.db_password)
cursor = conn.cursor()
wikipedia.output(u'Executing query:\n%s' % query)
query = query.encode(site.encoding())
cursor.execute(query)
while True:
try:
namespaceNumber, pageName = cursor.fetchone()
print namespaceNumber, pageName
except TypeError:
# Limit reached or no more results
break
#print pageName
if pageName:
namespace = site.namespace(namespaceNumber)
pageName = unicode(pageName, site.encoding())
if namespace:
pageTitle = '%s:%s' % (namespace, pageName)
else:
pageTitle = pageName
page = wikipedia.Page(site, pageTitle)
yield page
def YearPageGenerator(start = 1, end = 2050):
wikipedia.output(u"Starting with year %i" % start)
for i in xrange(start, end + 1):
if i % 100 == 0:
wikipedia.output(u'Preparing %i...' % i)
# There is no year 0
if i != 0:
current_year = date.formatYear(wikipedia.getSite().lang, i )
yield wikipedia.Page(wikipedia.getSite(), current_year)
def DayPageGenerator(startMonth=1, endMonth=12):
fd = date.FormatDate(wikipedia.getSite())
firstPage = wikipedia.Page(wikipedia.getSite(), fd(startMonth, 1))
wikipedia.output(u"Starting with %s" % firstPage.aslink())
for month in xrange(startMonth, endMonth+1):
for day in xrange(1, date.getNumberOfDaysInMonth(month)+1):
yield wikipedia.Page(wikipedia.getSite(), fd(month, day))
def NamespaceFilterPageGenerator(generator, namespaces):
"""
Wraps around another generator. Yields only those pages that are in a list
of specific namespace.
"""
for page in generator:
if page.namespace() in namespaces:
yield page
def RedirectFilterPageGenerator(generator):
"""
Wraps around another generator. Yields only those pages that are not redirects.
"""
for page in generator:
if not page.isRedirectPage():
yield page
def DuplicateFilterPageGenerator(generator):
"""
Wraps around another generator. Yields all pages, but prevents
duplicates.
"""
seenPages = []
for page in generator:
if page not in seenPages:
seenPages.append(page)
yield page
def CombinedPageGenerator(generators):
"""
Wraps around a list of other generators. Yields all pages generated by the
first generator; when the first generator stops yielding pages, yields those
generated by the second generator, etc.
"""
for generator in generators:
for page in generator:
yield page
def CategoryGenerator(generator):
"""
Wraps around another generator. Yields the same pages, but as Category
objects instead of Page objects. Makes sense only if it is ascertained
that only categories are being retrieved.
"""
for page in generator:
yield catlib.Category(page.site(), page.title())
def PageWithTalkPageGenerator(generator):
"""
Wraps around another generator. Yields the same pages, but for non-talk pages, it
also includes associated talk pages.
This generator does not check if the talk page in fact exists.
"""
for page in generator:
yield page
if not page.isTalkPage():
yield page.toggleTalkPage()
class _Preloader(threading.Thread):
def __init__(self, queue, generator, pageNumber):
threading.Thread.__init__(self)
self.queue = queue
self.generator = generator
self.pageNumber = pageNumber
# identification for debugging purposes
self.setName('Preloader-Thread')
# This thread dies when the main program terminates
self.setDaemon(True)
def preload(self, pages):
try:
while len(pages) > 0:
# It might be that the pages are on different sites,
# e.g. because the -interwiki parameter was used.
# Query the sites one by one.
site = pages[0].site()
pagesThisSite = [page for page in pages if page.site() == site]
pages = [page for page in pages if page.site() != site]
wikipedia.getall(site, pagesThisSite, throttle=False)
for page in pagesThisSite:
yield page
except IndexError:
# Can happen if the pages list is empty. Don't care.
pass
except wikipedia.SaxError:
# Ignore this error, and get the pages the traditional way later.
pass
def run(self):
try:
# this array will contain up to pageNumber pages and will be flushed
# after these pages have been preloaded and yielded.
somePages = []
for page in self.generator:
somePages.append(page)
# We don't want to load too many pages at once using XML export.
# We only get a maximum number at a time.
if len(somePages) >= self.pageNumber:
for refpage in self.preload(somePages):
self.queue.put(refpage)
somePages = []
if somePages:
# preload remaining pages
for refpage in self.preload(somePages):
self.queue.put(refpage)
self.queue.put(None) # to signal end of list
except Exception, e:
wikipedia.output(str(e))
self.queue.put(None) # to signal end of list
def PreloadingGenerator(generator, pageNumber=60):
"""
Yields the same pages as generator generator. Retrieves 60 pages (or
another number specified by pageNumber), loads them using
Special:Export, and yields them one after the other. Then retrieves more
pages, etc. Thus, it is not necessary to load each page separately.
Operates asynchronously, so the next batch of pages is loaded in the
background before the first batch is fully consumed.
"""
if pageNumber < 2:
raise ValueError("PreloadingGenerator needs to load more than 1 page.")
pagequeue = Queue.Queue(min(pageNumber//2, 10))
preloader = _Preloader(pagequeue, generator, pageNumber)
preloader.start()
while True:
# Queue.get() blocks the main thread. This means that the
# program wouldn't react to CTRL-C while it is waiting for
# a queue element.
# Thus, there is a timeout to the blocking, so that Python
# can check once a second if there is a KeyboardInterrupt.
try:
p = pagequeue.get(timeout = 1)
except Queue.Empty:
# This is expected. Keep waiting.
continue
if p is None:
return
yield p
class GeneratorFactory:
"""
This factory is responsible for processing command line arguments
that are used by many scripts and that determine on which pages
to work on.
"""
def __init__(self):
pass
def handleArg(self, arg):
gen = None
if arg.startswith('-filelinks'):
if len(arg) == 10:
fileLinksPageTitle = wikipedia.input(u'Links to which image page should be processed?')
else:
fileLinksPageTitle = arg[11:]
fileLinksPage = wikipedia.Page(wikipedia.getSite(), 'Image:' + fileLinksPageTitle)
gen = FileLinksGenerator(fileLinksPage)
elif arg.startswith('-unusedfiles'):
if len(arg) == 12:
gen = UnusedFilesGenerator()
else:
gen = UnusedFilesGenerator(number = int(arg[13:]))
elif arg.startswith('-unwatched'):
if len(arg) == 10:
gen = UnwatchedPagesPageGenerator()
else:
gen = UnwatchedPagesPageGenerator(number = int(arg[11:]))
elif arg.startswith('-withoutinterwiki'):
if len(arg) == 17:
gen = WithoutInterwikiPageGenerator()
else:
gen = WithoutInterwikiPageGenerator(number = int(arg[18:]))
elif arg.startswith('-interwiki'):
if len(arg) == 10:
title = wikipedia.input(u'Which page should be processed?')
else:
title = arg[11:]
page = wikipedia.Page(wikipedia.getSite(), title)
gen = InterwikiPageGenerator(page)
elif arg.startswith('-file'):
if len(arg) == 5:
textfilename = wikipedia.input(u'Please enter the local file name:')
else:
textfilename = arg[6:]
gen = TextfilePageGenerator(textfilename)
elif arg.startswith('-cat'):
if len(arg) == 4:
categoryname = wikipedia.input(u'Please enter the category name:')
else:
categoryname = arg[5:]
cat = catlib.Category(wikipedia.getSite(), 'Category:%s' % categoryname)
gen = CategorizedPageGenerator(cat)
elif arg.startswith('-subcat'):
if len(arg) == 7:
categoryname = wikipedia.input(u'Please enter the category name:')
else:
categoryname = arg[8:]
cat = catlib.Category(wikipedia.getSite(), 'Category:%s' % categoryname)
gen = CategorizedPageGenerator(cat, recurse = True)
elif arg.startswith('-ref'):
if len(arg) == 4:
referredPageTitle = wikipedia.input(u'Links to which page should be processed?')
else:
referredPageTitle = arg[5:]
referredPage = wikipedia.Page(wikipedia.getSite(), referredPageTitle)
gen = ReferringPageGenerator(referredPage)
elif arg.startswith('-links'):
if len(arg) == 6:
linkingPageTitle = wikipedia.input(u'Links from which page should be processed?')
else:
linkingPageTitle = arg[7:]
linkingPage = wikipedia.Page(wikipedia.getSite(), linkingPageTitle)
gen = LinkedPageGenerator(linkingPage)
elif arg.startswith('-weblink'):
if len(arg) == 8:
url = wikipedia.input(u'Pages with which weblink should be processed?')
else:
url = arg[9:]
gen = LinksearchPageGenerator(url)
elif arg.startswith('-transcludes'):
if len(arg) == len('-transcludes'):
transclusionPageTitle = wikipedia.input(u'Pages that transclude which page should be processed?')
else:
transclusionPageTitle = arg[len('-transcludes:'):]
transclusionPage = wikipedia.Page(wikipedia.getSite(), 'Template:%s' % transclusionPageTitle)
gen = ReferringPageGenerator(transclusionPage, onlyTemplateInclusion = True)
elif arg.startswith('-start'):
if len(arg) == 6:
firstPageTitle = wikipedia.input(u'At which page do you want to start?')
else:
firstPageTitle = arg[7:]
namespace = wikipedia.Page(wikipedia.getSite(), firstPageTitle).namespace()
firstPageTitle = wikipedia.Page(wikipedia.getSite(), firstPageTitle).titleWithoutNamespace()
gen = AllpagesPageGenerator(firstPageTitle, namespace, includeredirects = False)
elif arg.startswith('-new'):
if len(arg) >=5:
gen = NewpagesPageGenerator(number = int(arg[5:]))
else:
gen = NewpagesPageGenerator(number = 60)
elif arg.startswith('-google'):
if len(arg) == 8:
googleQuery = wikipedia.input(u'What do you want to search for?')
else:
googleQuery = arg[8:]
gen = GoogleSearchPageGenerator(googleQuery)
else:
return None
# make sure all yielded pages are unique
gen = DuplicateFilterPageGenerator(gen)
return gen
if __name__ == "__main__":
try:
gen = None
genFactory = GeneratorFactory()
for arg in wikipedia.handleArgs():
generator = genFactory.handleArg(arg)
if generator:
gen = generator
if gen:
for page in gen:
wikipedia.output(page.title(), toStdout = True)
else:
wikipedia.showHelp('pagegenerators')
finally:
wikipedia.stopme()