29 lines
557 B
Python
29 lines
557 B
Python
#!/usr/bin/env python
|
|
'''
|
|
Need a docstring.
|
|
'''
|
|
|
|
import argparse
|
|
from utils.helpers import (UrlPool, WebPage, sanitise_url, qualify_url)
|
|
|
|
def init_crawler(base_url=None):
|
|
'''
|
|
needs a docstring
|
|
'''
|
|
|
|
|
|
def run(args=None):
|
|
'''
|
|
needs a docstring.
|
|
'''
|
|
base_url = sanitise_url(args.url)
|
|
print(base_url)
|
|
|
|
if __name__ == '__main__':
|
|
|
|
parser = argparse.ArgumentParser(description='Recursive web crawler')
|
|
parser.add_argument("-u", "--url", required=True, help="Base url to crawl")
|
|
args = parser.parse_args()
|
|
|
|
run(args)
|