#import library
import urllib
import requests
import bs4
import pandas as pd
import re

# request a link
def request_url(link):
    response = requests.get(link)
    html = response.text
    return html

# Function to parse html
def parse_html(to_parse):
    soup = bs4.BeautifulSoup(to_parse, 'html.parser')
    return soup

#take sub menu news form inipasti.com
def all_section(main_url):
    section_list = []
    for i in main_url:
        soup = parse_html(request_url(i))
        for a in soup.find_all('a', href=True): 
            if a.text:
                section_list.append(a['href'])
    return section_list

def find_corona(main_url):
    url_list = []
    not_news=[]
    for i in main_url:
        if i.find('corona')!=-1 or i.find('covid')!=-1 or i.find('pandemi')!=-1 or i.find('psbb')!=-1 or i.find('social-distancing')!=-1:
            url_list.append(i)
    url_list = list(dict.fromkeys(url_list))
    return url_list

def writer(main_url):
    author = []
    for i in main_url:
        a = parse_html(request_url(i))
        try:
            b = a.find("div", class_ = "td-author-name vcard author")    
            if b:
                c = b.find("a")
                author.append(c.text)
            else:
                author.append('None')
        except :
            author.append('None')
            
    return author

def title(main_url):
    titles = []
    for i in main_url:
        soup = parse_html(request_url(i))
        try :
            h1 = soup.select('h1', {'class' : 'entry-title td-module-title'})[0].text.strip()
            titles.append(h1)
        except :
            titles.append('None')
    return titles

def date(main_url):
    dates = []
    for i in main_url:
        a = parse_html(request_url(i))
        try:
            b = a.find("span", class_ = "td-post-date").text
            c = re.sub("Inipasti.com - ", "", b)
            dates.append(c)
        except:
            dates.append('None')
    return dates

def collect_text(main_url, titles = [], author = [], dates = []):
    paragraf = []
    data = []
    join =[]
    isiteks = []
    #collect all text with tag <p> from articles    
    for i,j in enumerate(main_url):
        a = parse_html(request_url(j))
        try:
            content = a.find('div', class_='td-post-content')
            pragraf = content.find_all('p')
            for k in pragraf:
                paragraf.append(k.text)
            data.append(paragraf)
            paragraf = []
        except:
            data.append('None')
    for i in data:
        join.append(' '.join(i))
        
    #collect articles and urls in one variable  
    for i, j in enumerate(main_url):
        isiteks.append({'news' : 'Inipasti.com', 'link' : j, 'title' : titles[i], 'author' : author[i], 'date_time' : dates[i], 'paragraf' : join[i] })
    return isiteks

def get_news(main_url, file_name = ''):
    section = all_section(main_url)
    corona_news = find_corona(section)
    titles = title(corona_news)
    author = writer(corona_news)
    datetime = date(corona_news)
    text = collect_text(corona_news, titles, author, datetime)
    file = save_file(text, file_name)
    
    return file  

def save_file(file, file_name = ''):
    new_file = pd.DataFrame(file, columns=['news','link','title', 'author', 'date_time', 'paragraf'])
    new_file.to_csv(file_name + '.csv', index=True, encoding='utf-8', sep = ',')
    
    return new_file

main_url=["https://inipasti.com/"]

# r = get_news(main_url, "news_2")
# r

inipasti_news=get_news(main_url, file_name = 'inipasti_satu')
inipasti_news