Oni2/Validate%20External%20Links/validate_external_links.command

#!/bin/bash

# Runs the Validate External Links script
# Arguments: --help, --links URL, --exceptions URL, --output DIR, --record-ok-links, --show-added-slashes,
# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots-ng, --suggest-snapshots-ok,
# --check-archive-links, --take-screenshots FILE, --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
# Recommended rule:
# |----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ---|

# Set variables for any arguments we might use below
VALEXTLINKS="/path/to/Validate External Links/validate_external_links.sh"
LINKS_LOCAL="file:///path/to/Validate External Links/Sample files/extlinks.csv"
LINKS_ONLINE="https://wiki.oni2.net/w/extlinks.csv"
EXCEPT_LOCAL="file:///path/to/Validate External Links/exceptions.txt"
EXCEPT_ONLINE="https://wiki.oni2.net/Validate_External_Links/Exceptions"
REPORT_DIR="/path/to/where/you/want/Val Reports"
UPLOAD_INFO="/path/to/Validate External Links/sftp_login.txt"
CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"


# Run with end URL, record OK codes, and don't upload
#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70

# Run with start/end URLs, record OK codes, and don't upload
#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 1 --end-url 200

# Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload
#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 1810 --end-url 1815

# Run with unfinished screenshot feature, no upload
#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --take-screenshots "$CHROME"

# Normal run with no upload
bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10

# Normal run
#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --upload "$UPLOAD_INFO"
Revision:	1147
Committed:	Tue Feb 2 20:10:39 2021 UTC (4 years, 8 months ago) by iritscen
File size:	2340 byte(s)
Log Message:	ValExtLinks: Changed --suggest-snapshots to --suggest-snapshots-ng and added --suggest-snapshots-ok for getting snapshot URLs for all good links. This can be used to confirm that sites are backed up in case they die in the future, but note that this argument will take hours to run due to the API rate limit. Added awareness of API rate limit so Archive.org will not start blocking script.
#	Content
1	#!/bin/bash
2
3	# Runs the Validate External Links script
4	# Arguments: --help, --links URL, --exceptions URL, --output DIR, --record-ok-links, --show-added-slashes,
5	# --show-https-upgrades, --show-yt-redirects, --suggest-snapshots-ng, --suggest-snapshots-ok,
6	# --check-archive-links, --take-screenshots FILE, --timeout NUM, --start-url NUM, --end-url NUM, --upload FILE
7	# Recommended rule:
8	# \|----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ---\|
9
10	# Set variables for any arguments we might use below
11	VALEXTLINKS="/path/to/Validate External Links/validate_external_links.sh"
12	LINKS_LOCAL="file:///path/to/Validate External Links/Sample files/extlinks.csv"
13	LINKS_ONLINE="https://wiki.oni2.net/w/extlinks.csv"
14	EXCEPT_LOCAL="file:///path/to/Validate External Links/exceptions.txt"
15	EXCEPT_ONLINE="https://wiki.oni2.net/Validate_External_Links/Exceptions"
16	REPORT_DIR="/path/to/where/you/want/Val Reports"
17	UPLOAD_INFO="/path/to/Validate External Links/sftp_login.txt"
18	CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
19
20
21	# Run with end URL, record OK codes, and don't upload
22	#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --end-url 70
23
24	# Run with start/end URLs, record OK codes, and don't upload
25	#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --suggest-snapshots-ok --start-url 1 --end-url 200
26
27	# Run with local extlinks and exceptions, start/end URLs, record OK codes, and don't upload
28	#bash "$VALEXTLINKS" --links "$LINKS_LOCAL" --exceptions "$EXCEPT_LOCAL" --output "$REPORT_DIR" --record-ok-links --suggest-snapshots-ng --start-url 1810 --end-url 1815
29
30	# Run with unfinished screenshot feature, no upload
31	#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --record-ok-links --take-screenshots "$CHROME"
32
33	# Normal run with no upload
34	bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10
35
36	# Normal run
37	#bash "$VALEXTLINKS" --links "$LINKS_ONLINE" --exceptions "$EXCEPT_ONLINE" --output "$REPORT_DIR" --suggest-snapshots-ng --timeout 10 --upload "$UPLOAD_INFO"