scrapping berits cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import library\n",
    "import urllib\n",
    "import requests\n",
    "import bs4\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# request a link\n",
    "def request_url(link):\n",
    "    response = requests.get(link)\n",
    "    html = response.text\n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to parse html\n",
    "def parse_html(to_parse):\n",
    "    soup = bs4.BeautifulSoup(to_parse, 'html.parser')\n",
    "    return soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take sub menu news form liputan6.com\n",
    "def all_section(main_url):\n",
    "    section_list = []\n",
    "    for i in main_url:\n",
    "        soup = parse_html(request_url(i))\n",
    "        for a in soup.find_all('a', href=True): \n",
    "            if a.text:\n",
    "                section_list.append(a['href'])\n",
    "    return section_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get only link about corona or covid\n",
    "def find_corona(main_url):\n",
    "    url_list = []\n",
    "    not_news=[]\n",
    "    for i in main_url:\n",
    "        if i.find('corona')!=-1 or i.find('covid')!=-1:\n",
    "            # print(i)\n",
    "            url_list.append(i)\n",
    "    url_list = list(dict.fromkeys(url_list))\n",
    "    for i in url_list:\n",
    "        if (i.find('/tag/') != -1):\n",
    "            not_news.append(i)       \n",
    "    for j in not_news:\n",
    "        url_list.remove(j)\n",
    "    return url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def title(main_url):\n",
    "    titles = []\n",
    "    for i in main_url:\n",
    "        soup=parse_html(request_url(i))\n",
    "        get_title = soup.find(\"h1\", class_=\"title\")\n",
    "        if(get_title):\n",
    "            titles.append(' '.join(get_title.text.split()))\n",
    "        else:\n",
    "            titles.append('None')\n",
    "    return titles "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writer(main_url):\n",
    "    author = []\n",
    "    for i in main_url:\n",
    "        soup=parse_html(request_url(i))\n",
    "        try:\n",
    "            get_author = soup.find(\"div\", class_=\"detail_text\")\n",
    "            get_author1 = soup.find(\"div\", class_=\"date\")\n",
    "            get_author1 = ' '.join(get_author1.text.split())\n",
    "            get_author1 = get_author1.partition(\", \")[0]\n",
    "            if(get_author):\n",
    "                authorr = get_author.b.string\n",
    "                author.append(''.join(authorr.partition('(')[2:]).partition(\")\")[0])\n",
    "            elif(get_author1):\n",
    "                author.append(get_author1)\n",
    "            else:\n",
    "                author.append('None')\n",
    "        except:\n",
    "            author.append('None')\n",
    "    return author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def date(main_url):\n",
    "    datetime = []\n",
    "    for i in main_url:\n",
    "        soup=parse_html(request_url(i))\n",
    "        get_datetime = soup.find(\"div\", class_=\"date\")\n",
    "        if(get_datetime):\n",
    "            del_line = ' '.join(get_datetime.text.split())\n",
    "            datetime.append(''.join(del_line.partition('| ')[2:]))\n",
    "        else:\n",
    "            datetime.append('None')\n",
    "    return datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_text(main_url, titles = [], author = [], datetime = []):\n",
    "    paragraf = []\n",
    "    isiteks = []\n",
    "\n",
    "    for i in main_url:\n",
    "        a = parse_html(request_url(i))\n",
    "        try:\n",
    "            content = a.find('span', id=\"detikdetailtext\")\n",
    "            content1 = a.find(\"div\", class_=\"detail_text\")\n",
    "            content2 = a.find(\"div\", class_=\"content_detail\")\n",
    "            if(content):\n",
    "                s = ' '.join(content.text.split())\n",
    "                paragraf.append(s)\n",
    "            elif(content1):\n",
    "                s = ' '.join(content1.text.split())\n",
    "                paragraf.append(s.partition(\"[Gambas:Video CNN]\")[0])\n",
    "            elif(content2):\n",
    "                s = ' '.join(content2.text.split())\n",
    "                paragraf.append(s)\n",
    "            else:\n",
    "                paragraf.append('None')\n",
    "        except:\n",
    "            paragraf.append('None')           \n",
    "    \n",
    "    for i, j in enumerate(main_url):\n",
    "        isiteks.append({'news': 'CNN Indonesia', 'link' : j, 'title': titles[i], 'author' : author[i], 'date_time': datetime[i], 'paragraf' : paragraf[i]})\n",
    "    return isiteks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_file(file, file_name = ''):\n",
    "    new_file = pd.DataFrame(file, columns=['news','link','title', 'author', 'date_time', 'paragraf'])\n",
    "    new_file.to_csv(file_name + '.csv', index=True, encoding='utf-8', sep = ',')\n",
    "    \n",
    "    return new_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_news(main_url, file_name = ''):\n",
    "    section = all_section(main_url)\n",
    "    corona_news = find_corona(section)\n",
    "    titles = title(corona_news)\n",
    "    author = writer(corona_news)\n",
    "    datetime = date(corona_news)\n",
    "    text = collect_text(corona_news, titles, author, datetime)\n",
    "    file = save_file(text, file_name)\n",
    "    \n",
    "    return file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "url=[\"https://www.cnnindonesia.com\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news</th>\n",
       "      <th>link</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>date_time</th>\n",
       "      <th>paragraf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200505...</td>\n",
       "      <td>Mahasiswa PTN Bisa Minta Keringanan Uang Kulia...</td>\n",
       "      <td>fey/ain</td>\n",
       "      <td>Selasa, 05/05/2020 20:02 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Ketua Majelis Rektor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/internasional/202...</td>\n",
       "      <td>WNI Positif Corona di Luar Negeri 703 Orang, 3...</td>\n",
       "      <td>ayp</td>\n",
       "      <td>Selasa, 05/05/2020 19:52 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Kementerian Luar Neg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/internasional/202...</td>\n",
       "      <td>China Sebut RI Akan Menang Lawan Corona Secepa...</td>\n",
       "      <td>rds/dea</td>\n",
       "      <td>Selasa, 05/05/2020 14:57 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Pemerintah China mey...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200505...</td>\n",
       "      <td>Bertambah 243 Orang, Pasien Corona Sembuh Tert...</td>\n",
       "      <td>fra</td>\n",
       "      <td>Selasa, 05/05/2020 18:56 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Jumlah pasien positi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/ekonomi/202005051...</td>\n",
       "      <td>FOTO: Asa Tukang Cuci Mobil di Tengah Pandemi ...</td>\n",
       "      <td>CNN Indonesia/Andry Novelino</td>\n",
       "      <td>Selasa, 05/05/2020 18:47 WIB</td>\n",
       "      <td>Home Ekonomi Foto Bisnis FOTO: Asa Tukang Cuci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/internasional/202...</td>\n",
       "      <td>Corona Menurun, Pelajar Korsel Kembali Sekolah...</td>\n",
       "      <td>ans/ayp</td>\n",
       "      <td>Selasa, 05/05/2020 18:10 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Korea Selatan akan k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200505...</td>\n",
       "      <td>Pasien Covid-19 Klaster Ijtima Gowa Tulari Kel...</td>\n",
       "      <td>pnd/pmg</td>\n",
       "      <td>Selasa, 05/05/2020 19:32 WIB</td>\n",
       "      <td>Kendari, CNN Indonesia -- Dua pasien Covid-19 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/internasional/202...</td>\n",
       "      <td>VIDEO: AS Diklaim Sulit Temukan Warga Tanpa Ge...</td>\n",
       "      <td>AFP</td>\n",
       "      <td>Selasa, 05/05/2020 18:46 WIB</td>\n",
       "      <td>Home Internasional Video Eropa Amerika VIDEO: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/tv/20200505185659...</td>\n",
       "      <td>VIDEO: Gubernur Sumut Kunjungi RS Darurat Covi...</td>\n",
       "      <td>CNN Indonesia TV</td>\n",
       "      <td>Selasa, 05/05/2020 20:00 WIB</td>\n",
       "      <td>VIDEO: Gubernur Sumut Kunjungi RS Darurat Covi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/gaya-hidup/202005...</td>\n",
       "      <td>Jumisih Berpacu di Antara Ancaman PHK Buruh da...</td>\n",
       "      <td>NMA</td>\n",
       "      <td>Jumat, 01/05/2020 16:52 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Perawakan mungil tak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/olahraga/20200504...</td>\n",
       "      <td>Virus Corona dan Ujian Terberat Cristian Gonza...</td>\n",
       "      <td>TTF/har</td>\n",
       "      <td>Selasa, 05/05/2020 16:00 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Setelah menjadi mual...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200505...</td>\n",
       "      <td>2.200 TKI Akan Masuk Sumut, Antisipasi Covid-1...</td>\n",
       "      <td>fnr/pmg</td>\n",
       "      <td>Selasa, 05/05/2020 19:24 WIB</td>\n",
       "      <td>Medan, CNN Indonesia -- Pemerintah Provinsi Su...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200505...</td>\n",
       "      <td>Update Corona 5 Mei:12.071 Positif, 2.197 Oran...</td>\n",
       "      <td>yoa/ugo</td>\n",
       "      <td>Selasa, 05/05/2020 15:49 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Total jumlah pasien ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/ekonomi/202005030...</td>\n",
       "      <td>Perang Lawan Corona dan Benteng Terakhir Krisi...</td>\n",
       "      <td>Khudori</td>\n",
       "      <td>Minggu, 03/05/2020 10:04 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Resonansi krisis pan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/ekonomi/202004301...</td>\n",
       "      <td>Jalan Lapang Oligarki Ekonomi di Perppu Corona...</td>\n",
       "      <td>Arif Hulwan Muzayyin</td>\n",
       "      <td>Kamis, 30/04/2020 19:10 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Meski bisa menurunka...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://www.cnnindonesia.com/nasional/20200415...</td>\n",
       "      <td>Nina Bobo di Masa Corona</td>\n",
       "      <td>Yugo Hindarto</td>\n",
       "      <td>Kamis, 16/04/2020 08:05 WIB</td>\n",
       "      <td>Jakarta, CNN Indonesia -- Di masa pandemi coro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>CNN Indonesia</td>\n",
       "      <td>https://news.detik.com/berita/d-5003717/positi...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             news                                               link  \\\n",
       "0   CNN Indonesia  https://www.cnnindonesia.com/nasional/20200505...   \n",
       "1   CNN Indonesia  https://www.cnnindonesia.com/internasional/202...   \n",
       "2   CNN Indonesia  https://www.cnnindonesia.com/internasional/202...   \n",
       "3   CNN Indonesia  https://www.cnnindonesia.com/nasional/20200505...   \n",
       "4   CNN Indonesia  https://www.cnnindonesia.com/ekonomi/202005051...   \n",
       "5   CNN Indonesia  https://www.cnnindonesia.com/internasional/202...   \n",
       "6   CNN Indonesia  https://www.cnnindonesia.com/nasional/20200505...   \n",
       "7   CNN Indonesia  https://www.cnnindonesia.com/internasional/202...   \n",
       "8   CNN Indonesia  https://www.cnnindonesia.com/tv/20200505185659...   \n",
       "9   CNN Indonesia  https://www.cnnindonesia.com/gaya-hidup/202005...   \n",
       "10  CNN Indonesia  https://www.cnnindonesia.com/olahraga/20200504...   \n",
       "11  CNN Indonesia  https://www.cnnindonesia.com/nasional/20200505...   \n",
       "12  CNN Indonesia  https://www.cnnindonesia.com/nasional/20200505...   \n",
       "13  CNN Indonesia  https://www.cnnindonesia.com/ekonomi/202005030...   \n",
       "14  CNN Indonesia  https://www.cnnindonesia.com/ekonomi/202004301...   \n",
       "15  CNN Indonesia  https://www.cnnindonesia.com/nasional/20200415...   \n",
       "16  CNN Indonesia  https://news.detik.com/berita/d-5003717/positi...   \n",
       "\n",
       "                                                title  \\\n",
       "0   Mahasiswa PTN Bisa Minta Keringanan Uang Kulia...   \n",
       "1   WNI Positif Corona di Luar Negeri 703 Orang, 3...   \n",
       "2   China Sebut RI Akan Menang Lawan Corona Secepa...   \n",
       "3   Bertambah 243 Orang, Pasien Corona Sembuh Tert...   \n",
       "4   FOTO: Asa Tukang Cuci Mobil di Tengah Pandemi ...   \n",
       "5   Corona Menurun, Pelajar Korsel Kembali Sekolah...   \n",
       "6   Pasien Covid-19 Klaster Ijtima Gowa Tulari Kel...   \n",
       "7   VIDEO: AS Diklaim Sulit Temukan Warga Tanpa Ge...   \n",
       "8   VIDEO: Gubernur Sumut Kunjungi RS Darurat Covi...   \n",
       "9   Jumisih Berpacu di Antara Ancaman PHK Buruh da...   \n",
       "10  Virus Corona dan Ujian Terberat Cristian Gonza...   \n",
       "11  2.200 TKI Akan Masuk Sumut, Antisipasi Covid-1...   \n",
       "12  Update Corona 5 Mei:12.071 Positif, 2.197 Oran...   \n",
       "13  Perang Lawan Corona dan Benteng Terakhir Krisi...   \n",
       "14  Jalan Lapang Oligarki Ekonomi di Perppu Corona...   \n",
       "15                           Nina Bobo di Masa Corona   \n",
       "16                                               None   \n",
       "\n",
       "                          author                     date_time  \\\n",
       "0                        fey/ain  Selasa, 05/05/2020 20:02 WIB   \n",
       "1                            ayp  Selasa, 05/05/2020 19:52 WIB   \n",
       "2                        rds/dea  Selasa, 05/05/2020 14:57 WIB   \n",
       "3                            fra  Selasa, 05/05/2020 18:56 WIB   \n",
       "4   CNN Indonesia/Andry Novelino  Selasa, 05/05/2020 18:47 WIB   \n",
       "5                        ans/ayp  Selasa, 05/05/2020 18:10 WIB   \n",
       "6                        pnd/pmg  Selasa, 05/05/2020 19:32 WIB   \n",
       "7                            AFP  Selasa, 05/05/2020 18:46 WIB   \n",
       "8               CNN Indonesia TV  Selasa, 05/05/2020 20:00 WIB   \n",
       "9                            NMA   Jumat, 01/05/2020 16:52 WIB   \n",
       "10                       TTF/har  Selasa, 05/05/2020 16:00 WIB   \n",
       "11                       fnr/pmg  Selasa, 05/05/2020 19:24 WIB   \n",
       "12                       yoa/ugo  Selasa, 05/05/2020 15:49 WIB   \n",
       "13                       Khudori  Minggu, 03/05/2020 10:04 WIB   \n",
       "14          Arif Hulwan Muzayyin   Kamis, 30/04/2020 19:10 WIB   \n",
       "15                 Yugo Hindarto   Kamis, 16/04/2020 08:05 WIB   \n",
       "16                          None                          None   \n",
       "\n",
       "                                             paragraf  \n",
       "0   Jakarta, CNN Indonesia -- Ketua Majelis Rektor...  \n",
       "1   Jakarta, CNN Indonesia -- Kementerian Luar Neg...  \n",
       "2   Jakarta, CNN Indonesia -- Pemerintah China mey...  \n",
       "3   Jakarta, CNN Indonesia -- Jumlah pasien positi...  \n",
       "4   Home Ekonomi Foto Bisnis FOTO: Asa Tukang Cuci...  \n",
       "5   Jakarta, CNN Indonesia -- Korea Selatan akan k...  \n",
       "6   Kendari, CNN Indonesia -- Dua pasien Covid-19 ...  \n",
       "7   Home Internasional Video Eropa Amerika VIDEO: ...  \n",
       "8   VIDEO: Gubernur Sumut Kunjungi RS Darurat Covi...  \n",
       "9   Jakarta, CNN Indonesia -- Perawakan mungil tak...  \n",
       "10  Jakarta, CNN Indonesia -- Setelah menjadi mual...  \n",
       "11  Medan, CNN Indonesia -- Pemerintah Provinsi Su...  \n",
       "12  Jakarta, CNN Indonesia -- Total jumlah pasien ...  \n",
       "13  Jakarta, CNN Indonesia -- Resonansi krisis pan...  \n",
       "14  Jakarta, CNN Indonesia -- Meski bisa menurunka...  \n",
       "15  Jakarta, CNN Indonesia -- Di masa pandemi coro...  \n",
       "16                                               None  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnnindonesia_news=get_news(url, file_name = 'cnnindonesia_satu')\n",
    "cnnindonesia_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}