Python Scrapping Impossibile estrarre tutti i <li>

Question

import requests

from bs4 import BeautifulSoup


def get_data_from_web():
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    li = div.find_all('li')
    print(li)
    
get_data_from_web()

Sto cercando di estrarre le statistiche di Corona da http://mohfw.gov.in , ma ricevo solo un primo li

mentre ci sono in totale 3 li,

Ho provato dando lezioni specifiche per quei tag li ma sto ricevendo nonerisposta

<div class="col-xs-8 site-stats-count"> 
    <ul style="margin-top:0px;">
        <li class="bg-blue">
        <strong class="mob-hide">Active &nbsp;<span class="active_per"></span></strong>
        <strong class="mob-hide">973175<span class='up'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(14859<i class='fa fa-arrow-up'></i>)</span></strong>
        <!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->      
        <span class="mob-show">Active </span>
        <span class="mob-show"><span class="active_per"></span> </span> 
        <span class="mob-show"><strong>973175<span class='up'><br>(14859<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>  
        </li> 
        <li class="bg-green">
        <strong class="mob-hide">Discharged &nbsp;<span class="discharged_per"></span></strong>
        <strong class="mob-hide">3702595<span class='cup'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(78399<i class='fa fa-arrow-up'></i>)</span></strong>
        <span class="mob-show">Discharged </span>
        <span class="mob-show"><span class="discharged_per"></span> </span> 
        <span class="mob-show"><strong>3702595<span class='cup'><br>(78399<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>
        </li>                               
        <li class="bg-red">
        <strong class="mob-hide">Deaths &nbsp;<span class="death_per"></span></strong>
        <strong class="mob-hide">78586&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span class='up'>   (1114<i class='fa fa-arrow-up'></i>)</span></strong>
        <span class="mob-show">Deaths </span>
        <span class="mob-show"><span class="death_per"></span> </span>  
        <span class="mob-show"><strong>78586<span class='up'><br>(1114<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>
        <!--<span class='down'> <i class='fa fa-arrow-down'></i></span>-->      
        </li>
        </ul></div>

Answer 1

Il markup HTML su quella pagina è rotto, prova ad analizzarlo con lxmlo html5libparser:

import requests
from bs4 import BeautifulSoup


def get_data_from_web():
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'lxml')      # <-- change to lxml or html5lib
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    lis = div.find_all('li')
    for li in lis:
        print(li)
        print('-' * 80)

get_data_from_web()

Stampe:

<li class="bg-blue">
<strong class="mob-hide">Active  <span class="active_per"></span></strong>
<strong class="mob-hide">973175<span class="up">     (14859<i class="fa fa-arrow-up"></i>)</span></strong>
<!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->
<span class="mob-show">Active </span>
<span class="mob-show"><span class="active_per"></span> </span>
<span class="mob-show"><strong>973175<span class="up"><br/>(14859<i class="fa fa-arrow-up"></i>)</span></strong></span>
</li>
--------------------------------------------------------------------------------
<li class="bg-green">
<strong class="mob-hide">Discharged  <span class="discharged_per"></span></strong>
<strong class="mob-hide">3702595<span class="cup">     (78399<i class="fa fa-arrow-up"></i>)</span></strong>
<span class="mob-show">Discharged </span>
<span class="mob-show"><span class="discharged_per"></span> </span>
<span class="mob-show"><strong>3702595<span class="cup"><br/>(78399<i class="fa fa-arrow-up"></i>)</span></strong></span>
</li>
--------------------------------------------------------------------------------
<li class="bg-red">
<strong class="mob-hide">Deaths  <span class="death_per"></span></strong>
<strong class="mob-hide">78586     <span class="up">   (1114<i class="fa fa-arrow-up"></i>)</span></strong>
<span class="mob-show">Deaths </span>
<span class="mob-show"><span class="death_per"></span> </span>
<span class="mob-show"><strong>78586<span class="up"><br/>(1114<i class="fa fa-arrow-up"></i>)</span></strong></span>
<!--<span class='down'> <i class='fa fa-arrow-down'></i></span>-->
</li>
--------------------------------------------------------------------------------

Answer 2

Ho provato a ottenere le informazioni sul div e sembra che il div finisca con il primo tag li. Di seguito il codice. prova a eseguirlo una volta e vedrai.

import requests
from bs4 import BeautifulSoup

def get_data_from_web():
    print("here")
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    li = div.find_all('li')
    print(div)
    
get_data_from_web()

Ecco l'output:

<div class="col-xs-8 site-stats-count">
<ul style="margin-top:0px;">
<li class="bg-blue">
<strong class="mob-hide">Active  <span class="active_per"></span></strong>
<strong class="mob-hide">973175<span class="up">     (14859<i class="fa fa-arrow-up"></i>)</span></strong>
<!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->
<span class="mob-show">Active </span>
<span class="mob-show"><span class="active_per"></span> </span>
<span class="mob-show"><strong>973175<span class="up"><br/>(14859<i class="fa fa-arrow-up"></i>)</span></strong></span> </li></ul></div>