forked from adi0509/Python
-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrape.py
27 lines (17 loc) · 692 Bytes
/
scrape.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#This program scrapes all of the links of a given website.
from bs4 import BeautifulSoup
import requests
#Ask the user for the input URL for example https://www.census.gov/data/tables/2016/demo/popest/state-total.html
url = input("Enter a website to extract the URL's from: ")
#Request data from the server using GET protocol,
r = requests.get(url)
#In order to retrieve the data from the response object convert the raw response to text
data = r.text
#Pythons HTML parser
soup = BeautifulSoup(data, 'html.parser')
list =''
# Get all URLS from <a> tags with attribute href
for link in soup.find_all('a'):
print(link.get('href'))
#list += link.get('href') + '\n'
#print(list)