| 1 |
import os |
| 2 |
|
| 3 |
from urllib.parse import urljoin |
| 4 |
|
| 5 |
import pywikibot |
| 6 |
|
| 7 |
from pywikibot.bot import QuitKeyboardInterrupt |
| 8 |
from pywikibot import pagegenerators |
| 9 |
from pywikibot.comms.http import fetch |
| 10 |
from pywikibot.specialbots import UploadRobot |
| 11 |
from bs4 import BeautifulSoup |
| 12 |
|
| 13 |
first_run = False |
| 14 |
pages_checked = 0 |
| 15 |
oni2_images = 0 |
| 16 |
file_formats = ('.jpg', '.jpeg', '.png', '.gif', '.svg') |
| 17 |
|
| 18 |
# Scrapes the HTML at the given URL for image tags |
| 19 |
def get_image_links(url, shown): |
| 20 |
links = [] |
| 21 |
global oni2_images |
| 22 |
global pages_checked |
| 23 |
|
| 24 |
response = fetch(url) |
| 25 |
if response.status_code != 200: |
| 26 |
pywikibot.output('Skipping url: {}'.format(url)) |
| 27 |
return links |
| 28 |
|
| 29 |
soup = BeautifulSoup(response.text, 'html.parser') |
| 30 |
pages_checked = pages_checked + 1 |
| 31 |
if not shown: |
| 32 |
tagname = 'a' |
| 33 |
elif shown == 'just': |
| 34 |
tagname = 'img' |
| 35 |
else: |
| 36 |
tagname = ['a', 'img'] |
| 37 |
#pywikibot.output('Looking at tags.') |
| 38 |
for tag in soup.findAll(tagname): |
| 39 |
link = tag.get('src', tag.get('href', None)) |
| 40 |
if not link: |
| 41 |
#pywikibot.output('It is not a link.') |
| 42 |
continue |
| 43 |
#pywikibot.output('Got link {0}.'.format(link)) |
| 44 |
_, ext = os.path.splitext(link) |
| 45 |
if ext.lower() in file_formats: |
| 46 |
pywikibot.output('Found image link {0}.'.format(ext)) |
| 47 |
if "oni2.net" in link: |
| 48 |
pywikibot.stdout('Found an oni2.net image: {0}'.format(link)) |
| 49 |
oni2_images = oni2_images + 1 |
| 50 |
return links |
| 51 |
|
| 52 |
|
| 53 |
def main(*args): |
| 54 |
cat = '' |
| 55 |
url = '' |
| 56 |
image_url = False |
| 57 |
shown = False |
| 58 |
desc = [] |
| 59 |
|
| 60 |
local_args = pywikibot.handle_args(args) |
| 61 |
genFactory = pagegenerators.GeneratorFactory() |
| 62 |
|
| 63 |
for arg in local_args: |
| 64 |
if arg.startswith('-cat:'): |
| 65 |
cat = arg[5:] |
| 66 |
elif arg == '-shown': |
| 67 |
shown = True |
| 68 |
elif arg == '-justshown': |
| 69 |
shown = 'just' |
| 70 |
elif url == '': |
| 71 |
url = arg |
| 72 |
else: |
| 73 |
desc += [arg] |
| 74 |
desc = ' '.join(desc) |
| 75 |
|
| 76 |
site = pywikibot.Site() |
| 77 |
cat_obj = pywikibot.Category(site, cat) |
| 78 |
generator = pagegenerators.CategorizedPageGenerator(cat_obj, recurse=True) |
| 79 |
for page in pagegenerators.PreloadingGenerator(generator, 100): |
| 80 |
pywikibot.stdout('Checking page {0}'.format(page.title())) |
| 81 |
page_url = page.full_url().replace("%2F", "/") |
| 82 |
get_image_links(page_url, shown) |
| 83 |
|
| 84 |
global pages_checked |
| 85 |
global oni2_images |
| 86 |
pywikibot.stdout('Checked {0} page(s) and found {1} image(s) from oni2.net.'.format(pages_checked, oni2_images)) |
| 87 |
|
| 88 |
if __name__ == '__main__': |
| 89 |
main() |
