Github_Code_Analysis_Tool/temp.py at main · arijitde92/Github_Code_Analysis_Tool · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
from bs4 import BeautifulSoup
import requests

# lists
urls = []


# function created
def scrape(site):
    # getting the request from url
    r = requests.get(site)

    # converting the text
    s = BeautifulSoup(r.text, "html.parser")

    for i in s.find_all("a"):

        href = i.attrs['href']

        if href.startswith("/"):
            site = site + href

            if site not in urls:
                urls.append(site)
                print(site)
                # calling it self
                scrape(site)


# main function
if __name__ == "__main__":
    # website to be scrape
    site = "https://github.com/hhhrrrttt222111/developer-portfolio"

    # calling function
    scrape(site)