Python Scrapping Невозможно извлечь все <li>

Question

import requests

from bs4 import BeautifulSoup


def get_data_from_web():
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    li = div.find_all('li')
    print(li)
    
get_data_from_web()

Я пытаюсь извлечь статистику Corona из http://mohfw.gov.in , но я получаю только один первый лай

пока всего 3 ли,

Я попытался дать класс специально для этих тегов li, но получаю noneответ

<div class="col-xs-8 site-stats-count"> 
    <ul style="margin-top:0px;">
        <li class="bg-blue">
        <strong class="mob-hide">Active &nbsp;<span class="active_per"></span></strong>
        <strong class="mob-hide">973175<span class='up'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(14859<i class='fa fa-arrow-up'></i>)</span></strong>
        <!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->      
        <span class="mob-show">Active </span>
        <span class="mob-show"><span class="active_per"></span> </span> 
        <span class="mob-show"><strong>973175<span class='up'><br>(14859<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>  
        </li> 
        <li class="bg-green">
        <strong class="mob-hide">Discharged &nbsp;<span class="discharged_per"></span></strong>
        <strong class="mob-hide">3702595<span class='cup'>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(78399<i class='fa fa-arrow-up'></i>)</span></strong>
        <span class="mob-show">Discharged </span>
        <span class="mob-show"><span class="discharged_per"></span> </span> 
        <span class="mob-show"><strong>3702595<span class='cup'><br>(78399<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>
        </li>                               
        <li class="bg-red">
        <strong class="mob-hide">Deaths &nbsp;<span class="death_per"></span></strong>
        <strong class="mob-hide">78586&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span class='up'>   (1114<i class='fa fa-arrow-up'></i>)</span></strong>
        <span class="mob-show">Deaths </span>
        <span class="mob-show"><span class="death_per"></span> </span>  
        <span class="mob-show"><strong>78586<span class='up'><br>(1114<i class='fa fa-arrow-up'></i>)</span></strong></span> </span>
        <!--<span class='down'> <i class='fa fa-arrow-down'></i></span>-->      
        </li>
        </ul></div>

Answer 1

Разметка HTML на этой странице повреждена, попробуйте проанализировать ее с помощью lxmlили html5libпарсеров:

import requests
from bs4 import BeautifulSoup


def get_data_from_web():
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'lxml')      # <-- change to lxml or html5lib
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    lis = div.find_all('li')
    for li in lis:
        print(li)
        print('-' * 80)

get_data_from_web()

Печать:

<li class="bg-blue">
<strong class="mob-hide">Active  <span class="active_per"></span></strong>
<strong class="mob-hide">973175<span class="up">     (14859<i class="fa fa-arrow-up"></i>)</span></strong>
<!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->
<span class="mob-show">Active </span>
<span class="mob-show"><span class="active_per"></span> </span>
<span class="mob-show"><strong>973175<span class="up"><br/>(14859<i class="fa fa-arrow-up"></i>)</span></strong></span>
</li>
--------------------------------------------------------------------------------
<li class="bg-green">
<strong class="mob-hide">Discharged  <span class="discharged_per"></span></strong>
<strong class="mob-hide">3702595<span class="cup">     (78399<i class="fa fa-arrow-up"></i>)</span></strong>
<span class="mob-show">Discharged </span>
<span class="mob-show"><span class="discharged_per"></span> </span>
<span class="mob-show"><strong>3702595<span class="cup"><br/>(78399<i class="fa fa-arrow-up"></i>)</span></strong></span>
</li>
--------------------------------------------------------------------------------
<li class="bg-red">
<strong class="mob-hide">Deaths  <span class="death_per"></span></strong>
<strong class="mob-hide">78586     <span class="up">   (1114<i class="fa fa-arrow-up"></i>)</span></strong>
<span class="mob-show">Deaths </span>
<span class="mob-show"><span class="death_per"></span> </span>
<span class="mob-show"><strong>78586<span class="up"><br/>(1114<i class="fa fa-arrow-up"></i>)</span></strong></span>
<!--<span class='down'> <i class='fa fa-arrow-down'></i></span>-->
</li>
--------------------------------------------------------------------------------

Answer 2

Я попытался получить информацию о div, и кажется, что div заканчивается первым тегом li. Ниже приведен код. попробуйте запустить его один раз, и вы увидите.

import requests
from bs4 import BeautifulSoup

def get_data_from_web():
    print("here")
    url = "http://mohfw.gov.in"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    div = soup.find('div', class_='col-xs-8 site-stats-count')
    li = div.find_all('li')
    print(div)
    
get_data_from_web()

Вот результат -

<div class="col-xs-8 site-stats-count">
<ul style="margin-top:0px;">
<li class="bg-blue">
<strong class="mob-hide">Active  <span class="active_per"></span></strong>
<strong class="mob-hide">973175<span class="up">     (14859<i class="fa fa-arrow-up"></i>)</span></strong>
<!--<span class='down'>3565 <i class='fa fa-arrow-down'></i></span>-->
<span class="mob-show">Active </span>
<span class="mob-show"><span class="active_per"></span> </span>
<span class="mob-show"><strong>973175<span class="up"><br/>(14859<i class="fa fa-arrow-up"></i>)</span></strong></span> </li></ul></div>